From ee3ef536716004a31040d695a54bc09e4c224ab1 Mon Sep 17 00:00:00 2001 From: Anna Hoffman Date: Fri, 24 Jul 2026 10:08:15 -0400 Subject: [PATCH 1/5] Refresh vector search samples for March 2026 product behavior - Quickstart scripts (diskann-quickstart-*) updated to the current VECTOR_SEARCH ... WITH APPROXIMATE syntax; add filler rows to satisfy the 100-row CREATE VECTOR INDEX minimum; remove the obsolete ALLOW_STALE_VECTOR_INDEX + rebuild-index-after-DML workflow. - Improvements script: rename approximate_staleness_percent -> graph_catchup_pending_percent (actual DMV name in the current build); safer IF EXISTS external-model drop; fix two syntax typos. - Wikipedia/006-fulltext-setup.sql: clearer full-text population timing note. - Wikipedia/008-semantic-reranking.sql + Semantic-Reranking/02-rerank.sql: Cohere Foundry deployment naming (v4.0 -> v4-0, no dots), trim payload to top-5 with LEFT(text,150) to fit GlobalStandard 1000 tokens/min cap. - Wikipedia/009-wikipedia-fp16.sql: idempotent DDL, cleaner rewrite. - Hybrid-Search/hybrid_search.py: fix CAST(? AS VECTOR) binding bug - double CAST(CAST(? AS NVARCHAR(MAX)) AS VECTOR(...)) is required with the ODBC driver's ntext binding. - RAG-with-Documents notebook: switch from pyodbc + manual Entra token struct to first-party mssql-python with authentication='ActiveDirectory Default'. Companion requirements.txt, .env.sample, Readme.md updates. - New sample: DiskANN/FineFoodReviews/ - end-to-end hybrid search over the Amazon Fine Food Reviews dataset (dataset already shipped in Datasets/finefoodembeddings.csv). --- .gitignore | 6 + DiskANN/FineFoodReviews/000-setup.sql | 64 + .../FineFoodReviews/001-load-and-embed.sql | 74 + .../002-diskann-and-fulltext.sql | 80 + DiskANN/FineFoodReviews/003-hybrid-search.sql | 134 ++ DiskANN/FineFoodReviews/README.md | 29 + DiskANN/FineFoodReviews/_embed-reviews.py | 45 + DiskANN/FineFoodReviews/_load-reviews.py | 74 + DiskANN/Wikipedia/006-fulltext-setup.sql | 4 +- DiskANN/Wikipedia/008-semantic-reranking.sql | 31 +- DiskANN/Wikipedia/009-wikipedia-fp16.sql | 326 ++-- ...kann-quickstart-azure-sql-improvements.sql | 24 +- DiskANN/diskann-quickstart-azure-sql.sql | 90 +- .../diskann-quickstart-sql-server-2025.sql | 28 +- Hybrid-Search/hybrid_search.py | 7 +- RAG-with-Documents/.env.sample | 6 +- RAG-with-Documents/RAG-with-resumes.ipynb | 1500 ++++++++++++----- RAG-with-Documents/Readme.md | 2 +- RAG-with-Documents/requirements.txt | 2 +- README.md | 2 + Semantic-Reranking/02-rerank.sql | 30 +- 21 files changed, 1929 insertions(+), 629 deletions(-) create mode 100644 DiskANN/FineFoodReviews/000-setup.sql create mode 100644 DiskANN/FineFoodReviews/001-load-and-embed.sql create mode 100644 DiskANN/FineFoodReviews/002-diskann-and-fulltext.sql create mode 100644 DiskANN/FineFoodReviews/003-hybrid-search.sql create mode 100644 DiskANN/FineFoodReviews/README.md create mode 100644 DiskANN/FineFoodReviews/_embed-reviews.py create mode 100644 DiskANN/FineFoodReviews/_load-reviews.py diff --git a/.gitignore b/.gitignore index c263db2..cf1568e 100644 --- a/.gitignore +++ b/.gitignore @@ -406,3 +406,9 @@ FodyWeavers.xsd /DotNet/SqlClient/Properties/launchSettings Empty.json /DotNet/SqlClient/Properties/launchSettingsProd.json /DotNet/SqlClient/Properties/launchSettings.prod.json + +# macOS +.DS_Store + +# Local session prep (VSLive 2026 · Beyond Embeddings) +VSLive2026/ diff --git a/DiskANN/FineFoodReviews/000-setup.sql b/DiskANN/FineFoodReviews/000-setup.sql new file mode 100644 index 0000000..22fd230 --- /dev/null +++ b/DiskANN/FineFoodReviews/000-setup.sql @@ -0,0 +1,64 @@ +/* + Fine Food Reviews · 000-setup + ------------------------------------------ + Creates the reviews table + external model + external table for CSV load. + + Pre-req: PREVIEW_FEATURES = ON at the database scope. + Full-text search enabled on the database. + An Azure OpenAI text-embedding-3-small deployment reachable from the DB + via database-scoped credential + external URL endpoint. +*/ +-- Uncomment if using SQL Server 2025: +-- use FineFoodReviews; +-- go + +-- ---------- reviews table ---------- +drop table if exists dbo.reviews; +go + +create table dbo.reviews ( + Id int not null primary key, + Time bigint null, + ProductId nvarchar(50) null, + UserId nvarchar(50) null, + Score tinyint null, -- 1..5 + Summary nvarchar(500) null, -- short review title + [Text] nvarchar(max) null, -- long review body + combined as (isnull(Summary,'') + N': ' + isnull([Text],'')) persisted, + embedding vector(1536) null -- populated by 001-load-and-embed.sql +); +go + +-- ---------- external model (reuse if already present) ---------- +if not exists (select 1 from sys.external_models where name = N'AIEmbeddings') +begin + print N'AIEmbeddings external model not found. Create it via the pattern in ../Wikipedia/001-setup-objects.sql'; + -- Example (adjust endpoint, deployment name, and credential to your Azure OpenAI resource): + /* + create external model AIEmbeddings + with ( + location = 'https://.openai.azure.com/openai/deployments/text-embedding-3-small/embeddings?api-version=2024-08-01-preview', + api_format = 'Azure OpenAI', + model_type = embeddings, + model = 'text-embedding-3-small', + credential = [https://.openai.azure.com] + ); + */ +end; +go + +-- ---------- (optional) external data source for CSV load ---------- +-- Only needed if loading from Azure Blob Storage. Skip if loading via SqlBulkCopy from a client. +-- +-- create database scoped credential [sample_data] +-- with identity = 'Managed Identity'; +-- go +-- create external data source [sample_data] +-- with ( +-- type = blob_storage, +-- location = 'https://.blob.core.windows.net/sample-data/', +-- credential = [sample_data] +-- ); +-- go + +print N'Setup complete. Next: 001-load-and-embed.sql'; diff --git a/DiskANN/FineFoodReviews/001-load-and-embed.sql b/DiskANN/FineFoodReviews/001-load-and-embed.sql new file mode 100644 index 0000000..faecf9c --- /dev/null +++ b/DiskANN/FineFoodReviews/001-load-and-embed.sql @@ -0,0 +1,74 @@ +/* + Fine Food Reviews · 001-load-and-embed + --------------------------------------------------- + Loads the first 500 rows of Datasets/Reviews.csv into dbo.reviews, then + generates a VECTOR(1536) embedding for each row via AI_GENERATE_EMBEDDINGS. + + Load options: + (A) Client-side load (RECOMMENDED for the demo — no blob storage needed). + Use SqlBulkCopy / bcp / Azure Data Studio's Import wizard to push + Datasets/Reviews.csv (first 500 rows) into dbo.reviews. Then jump to + "STEP 2: EMBED" below. + + (B) Server-side load from Azure Blob Storage. Uncomment the BULK INSERT + block below if you've staged the CSV in the external data source + declared in 000-setup.sql. +*/ +-- Uncomment if using SQL Server 2025: +-- use FineFoodReviews; +-- go + +-- ========================================================================== +-- STEP 1 · LOAD (option B — server-side; keep commented if using option A) +-- ========================================================================== +/* +truncate table dbo.reviews; + +bulk insert dbo.reviews (Id, Time, ProductId, UserId, Score, Summary, [Text]) +from 'reviews/Reviews.csv' +with ( + data_source = 'sample_data', + format = 'csv', + firstrow = 2, + codepage = '65001', + fieldterminator = ',', + rowterminator = '0x0a', + fieldquote = '"', + batchsize = 500, + lastrow = 501, -- keep it small for a live demo + tablock +); +go +*/ + +-- ========================================================================== +-- STEP 2 · EMBED +-- ========================================================================== +-- Confirm row count +select count(*) as loaded_rows, + sum(case when [Text] is not null then 1 else 0 end) as with_text +from dbo.reviews; +go + +-- Sanity check the external model +select top (1) + ai_generate_embeddings(N'hello world' use model AIEmbeddings) as sample_embedding_bytes; +go + +-- Bulk-embed all reviews. On 500 rows this is ~30 s over the network. +update dbo.reviews +set embedding = ai_generate_embeddings([combined] use model AIEmbeddings) +where embedding is null + and [combined] is not null; +go + +-- Verify: every row has an embedding +select + count(*) as total_rows, + sum(iif(embedding is null, 1, 0)) as missing_embeddings, + min(datalength(embedding)) as min_bytes, + max(datalength(embedding)) as max_bytes -- expected: 6152 bytes (1536 × 4 + 8-byte header) +from dbo.reviews; +go + +print N'Load + embed complete. Next: 002-diskann-and-fulltext.sql'; diff --git a/DiskANN/FineFoodReviews/002-diskann-and-fulltext.sql b/DiskANN/FineFoodReviews/002-diskann-and-fulltext.sql new file mode 100644 index 0000000..4f416d9 --- /dev/null +++ b/DiskANN/FineFoodReviews/002-diskann-and-fulltext.sql @@ -0,0 +1,80 @@ +/* + Fine Food Reviews · 002-diskann-and-fulltext + --------------------------------------------------------- + Builds the two indexes hybrid search needs: + 1. DiskANN vector index on dbo.reviews(embedding) — for VECTOR_SEARCH + 2. Full-text catalog + index on dbo.reviews(combined) — for FREETEXTTABLE + + Both indexes live on the same table. No data movement. +*/ +-- Uncomment if using SQL Server 2025: +-- use FineFoodReviews; +-- go + +-- ========================================================================== +-- 1 · DiskANN vector index +-- ========================================================================== +if exists ( + select 1 + from sys.indexes + where object_id = object_id(N'dbo.reviews') and name = N'vec_idx_reviews' +) + drop index vec_idx_reviews on dbo.reviews; +go + +create vector index vec_idx_reviews +on dbo.reviews (embedding) +with ( + metric = 'cosine', + type = 'diskann' +); +go + +-- Verify +select + i.name as index_name, + v.build_parameters, + json_value(v.build_parameters, '$.Version') as index_version +from sys.vector_indexes v +join sys.indexes i on v.object_id = i.object_id and v.index_id = i.index_id +where v.object_id = object_id(N'dbo.reviews'); +go + +-- ========================================================================== +-- 2 · Full-text catalog + index on combined (Summary + ': ' + Text) +-- ========================================================================== +if exists (select 1 from sys.fulltext_catalogs where name = N'ft_reviews_catalog') +begin + if exists ( + select 1 from sys.fulltext_indexes + where object_id = object_id(N'dbo.reviews') + ) + drop fulltext index on dbo.reviews; + + drop fulltext catalog ft_reviews_catalog; +end; +go + +create fulltext catalog ft_reviews_catalog as default; +go + +create fulltext index on dbo.reviews (combined language 1033) + key index PK__reviews -- adjust if the primary-key index has a different auto-name + on ft_reviews_catalog + with change_tracking auto; +go + +-- Wait for population (500 rows populates in ~10s). Verify: +-- 0 = idle (populated); non-zero = still crawling. +select + fulltextcatalogproperty(N'ft_reviews_catalog', 'PopulateStatus') as populate_status, + fulltextcatalogproperty(N'ft_reviews_catalog', 'ItemCount') as item_count; +go + +-- Smoke test full-text: how many reviews mention "oatmeal"? +select count(*) as oatmeal_mentions +from dbo.reviews +where contains(combined, N'oatmeal'); +go + +print N'Indexes ready. Next: 003-hybrid-search.sql'; diff --git a/DiskANN/FineFoodReviews/003-hybrid-search.sql b/DiskANN/FineFoodReviews/003-hybrid-search.sql new file mode 100644 index 0000000..5da8d25 --- /dev/null +++ b/DiskANN/FineFoodReviews/003-hybrid-search.sql @@ -0,0 +1,134 @@ +/* + Fine Food Reviews · 003-hybrid-search + --------------------------------------------------- + Three queries, back to back: + Q1 vector-only — great at paraphrase, weak at SKUs / brand names + Q2 full-text only — great at exact matches, weak at paraphrase + Q3 HYBRID (RRF) — both. Rows that win in either modality surface. + + Same table, same query, three retrieval strategies. This is the whole point. +*/ +-- Uncomment if using SQL Server 2025: +-- use FineFoodReviews; +-- go + +set statistics time on; +set statistics io on; +go + +-- -------------------------------------------------------------------------- +-- Q1 · VECTOR-ONLY (paraphrase query) +-- -------------------------------------------------------------------------- +-- Shopper intent: "smooth flavorful coffee" +-- Reviewers rarely say "smooth" — they say "not bitter", "not burnt", "mellow", +-- "easy on the stomach". Vectors bridge that. BM25 alone would miss most of it. +declare @q1 nvarchar(500) = N'smooth flavorful coffee'; +declare @v1 vector(1536) = ai_generate_embeddings(@q1 use model AIEmbeddings); + +select top (10) with approximate + r.Id, + r.ProductId, + r.Score, + r.Summary, + left(r.[Text], 120) + iif(len(r.[Text]) > 120, N'…', N'') as text_preview, + s.distance as cosine_distance +from vector_search( + table = dbo.reviews as r, + column = embedding, + similar_to = @v1, + metric = 'cosine' +) as s +order by s.distance; +go + + +-- -------------------------------------------------------------------------- +-- Q2 · FULL-TEXT ONLY (brand / SKU query) +-- -------------------------------------------------------------------------- +-- Shopper intent: "Green Mountain Nantucket Blend" — an exact brand + product name. +-- Vector alone drifts to generic Keurig / coffee reviews. +-- BM25 nails the literal brand + blend match. +select top (10) + r.Id, + r.ProductId, + r.Score, + r.Summary, + left(r.[Text], 120) + iif(len(r.[Text]) > 120, N'…', N'') as text_preview, + ftt.[rank] as bm25_rank +from dbo.reviews r +inner join freetexttable(dbo.reviews, combined, N'Green Mountain Nantucket Blend') ftt + on r.Id = ftt.[KEY] +order by ftt.[rank] desc; +go + +-- -------------------------------------------------------------------------- +-- Q3 · HYBRID (vector + full-text, blended with RRF) +-- -------------------------------------------------------------------------- +-- Shopper intent: "Green Mountain blend that is smooth and flavorful" — brand AND paraphrase. +-- Keyword side locks onto "Green Mountain"; vector side finds reviewers writing +-- "not bitter", "mellow", "easy on the stomach". RRF surfaces rows that win in +-- either signal. +declare @q nvarchar(1000) = N'Green Mountain blend that is smooth and flavorful'; +declare @v vector(1536) = ai_generate_embeddings(@q use model AIEmbeddings); +declare @k int = 20; -- candidates per signal +declare @rrf_k int = 60; -- standard RRF constant + +with keyword_search as ( + select top (@k) + r.Id, + rank() over (order by ftt.[rank] desc) as keyword_rank + from dbo.reviews r + inner join freetexttable(dbo.reviews, combined, @q) ftt + on r.Id = ftt.[KEY] + order by ftt.[rank] desc +), +semantic_search as ( + select top (@k) + s.Id, + rank() over (order by s.cosine_distance) as vector_rank + from ( + select top (@k) with approximate + r.Id, + s0.distance as cosine_distance + from vector_search( + table = dbo.reviews as r, + column = embedding, + similar_to = @v, + metric = 'cosine' + ) as s0 + order by s0.distance + ) s +), +fused as ( + select + coalesce(ss.Id, ks.Id) as Id, + ss.vector_rank, + ks.keyword_rank, + coalesce(1.0 / (@rrf_k + ss.vector_rank), 0.0) + + coalesce(1.0 / (@rrf_k + ks.keyword_rank), 0.0) as rrf_score + from semantic_search ss + full outer join keyword_search ks on ss.Id = ks.Id +) +select top (7) + r.Id, + r.ProductId, + r.Score, + r.Summary, + left(r.[Text], 100) + iif(len(r.[Text]) > 100, N'…', N'') as text_preview, + rank() over (order by f.rrf_score desc) as rrf_rank, + f.vector_rank, + f.keyword_rank, + cast(f.rrf_score * 1000 as int) as rrf_score_x1000 +from fused f +inner join dbo.reviews r on f.Id = r.Id +order by f.rrf_score desc; +go + +/* + Takeaway + --------- + Hybrid isn't a library. It's a WITH clause. + Both indexes live on the same table. No second data store. + Rerank is one REST call away when relevance beats latency — see + ../../Semantic-Reranking/02-rerank.sql. +*/ diff --git a/DiskANN/FineFoodReviews/README.md b/DiskANN/FineFoodReviews/README.md new file mode 100644 index 0000000..d63d505 --- /dev/null +++ b/DiskANN/FineFoodReviews/README.md @@ -0,0 +1,29 @@ +# Fine Food Reviews — Hybrid Search + +Hybrid vector + full-text search over Amazon Fine Food Reviews. Runs on **Azure SQL** (Hyperscale or General Purpose) and **SQL Server 2025**. + +**Scenario:** e-commerce shoppers type BOTH paraphrases (*"healthy oatmeal for a picky eater"*) and product SKUs / brand names (*"Merrick Turducken"*). Vector search misses SKUs; full-text misses paraphrases. Hybrid reciprocal-rank fusion (RRF) surfaces rows that win in either signal. + +## Files (run in order once, then re-run only `003` to compare queries) + +| # | File | Purpose | +|---|---|---| +| 000 | [`000-setup.sql`](000-setup.sql) | Create the `reviews` table, external model, and (optionally) the source-data external table. | +| 001 | [`001-load-and-embed.sql`](001-load-and-embed.sql) | Load 500 sample reviews from [`../../Datasets/Reviews.csv`](../../Datasets/Reviews.csv), then embed via `AI_GENERATE_EMBEDDINGS`. | +| 002 | [`002-diskann-and-fulltext.sql`](002-diskann-and-fulltext.sql) | Build the DiskANN vector index + the full-text catalog/index on `combined`. | +| 003 | [`003-hybrid-search.sql`](003-hybrid-search.sql) | Vector-only, full-text-only, and hybrid-RRF queries side by side. | + +Data source: [`../../Datasets/Reviews.csv`](../../Datasets/Reviews.csv) (500k Amazon Fine Food Reviews; the sample uses the first 500 rows). Pre-computed embeddings alternative: [`../../Datasets/FineFoodEmbeddings.csv`](../../Datasets/FineFoodEmbeddings.csv). + +## Timing on a 2-vCore Hyperscale instance (500 rows) + +- Embedding 500 rows: ~30 s (network to Azure OpenAI). +- DiskANN build: ~2 s. +- Full-text catalog build: ~10 s. +- Hybrid query: ~150 ms. + +## Pre-reqs + +- An `AIEmbeddings` external model pointing at a `text-embedding-3-small` deployment. See [`../Wikipedia/001-setup-objects.sql`](../Wikipedia/001-setup-objects.sql) for the setup pattern. +- Full-text search enabled on the database. +- Optional Python helpers ([`_load-reviews.py`](_load-reviews.py), [`_embed-reviews.py`](_embed-reviews.py)) for loading the CSV directly from Python instead of via `OPENROWSET`. diff --git a/DiskANN/FineFoodReviews/_embed-reviews.py b/DiskANN/FineFoodReviews/_embed-reviews.py new file mode 100644 index 0000000..3287fa1 --- /dev/null +++ b/DiskANN/FineFoodReviews/_embed-reviews.py @@ -0,0 +1,45 @@ +"""Batch-embed dbo.reviews in chunks of 50 to avoid REST endpoint timeout. +Uses mssql-python (the native Microsoft first-party Python driver).""" +import sys, time +import mssql_python + +SERVER = "antho-test-server.database.windows.net" +DATABASE = "VSLive2026" + +def connect(): + return mssql_python.connect( + server=SERVER, + database=DATABASE, + authentication="ActiveDirectoryDefault", + ) + +conn = connect() +cur = conn.cursor() + +# Set a long command timeout since AI_GENERATE_EMBEDDINGS is external +cur.execute("SET LOCK_TIMEOUT 60000;") + +start = time.time() +BATCH = 50 +for lo in range(1, 501, BATCH): + hi = min(lo + BATCH - 1, 500) + for attempt in range(3): + try: + cur.execute(f""" + UPDATE dbo.reviews + SET embedding = AI_GENERATE_EMBEDDINGS([combined] USE MODEL AIEmbeddings) + WHERE embedding IS NULL + AND [combined] IS NOT NULL + AND Id BETWEEN {lo} AND {hi}; + """) + conn.commit() + break + except mssql_python.Error as e: + print(f" retry {attempt+1} on batch {lo}-{hi}: {str(e)[:120]}") + time.sleep(3) + cur.execute("SELECT SUM(IIF(embedding IS NOT NULL,1,0)) FROM dbo.reviews") + total = cur.fetchone()[0] + print(f"[batch {lo:3d}-{hi:3d}] embedded so far = {total} / 500 (elapsed {time.time()-start:.1f}s)") + +conn.close() +print("[done]") diff --git a/DiskANN/FineFoodReviews/_load-reviews.py b/DiskANN/FineFoodReviews/_load-reviews.py new file mode 100644 index 0000000..bd7791f --- /dev/null +++ b/DiskANN/FineFoodReviews/_load-reviews.py @@ -0,0 +1,74 @@ +"""Load first 500 rows of Datasets/Reviews.csv into dbo.reviews on VSLive2026, +then trigger AI_GENERATE_EMBEDDINGS server-side. Uses mssql-python (the native +Microsoft first-party Python driver) with Entra Default auth — no token juggling.""" +import csv, sys, time +from pathlib import Path +import mssql_python + +SERVER = "antho-test-server.database.windows.net" +DATABASE = "VSLive2026" +CSV = Path("/Users/annahoffman/azure-sql-db-vector-search/Datasets/Reviews.csv") +N_ROWS = 500 + +def connect(): + return mssql_python.connect( + server=SERVER, + database=DATABASE, + authentication="ActiveDirectoryDefault", + ) + +conn = None +try: + print("[auth] connecting via mssql-python + Entra Default") + conn = connect() + print("[auth] connected") +except Exception as e: + print(f"[auth] failed: {e}") + sys.exit(1) + +cur = conn.cursor() + +# Reset + load +print(f"[load] reading first {N_ROWS} rows from {CSV.name}") +rows = [] +with CSV.open(newline="", encoding="utf-8") as f: + reader = csv.reader(f) + header = next(reader) + # Expected columns from Amazon Fine Food Reviews: + # Id, ProductId, UserId, ProfileName, HelpfulnessNumerator, HelpfulnessDenominator, Score, Time, Summary, Text + print(f"[load] header = {header}") + idx = {name: i for i, name in enumerate(header)} + for i, r in enumerate(reader): + if i >= N_ROWS: + break + rows.append(( + int(r[idx["Id"]]), + int(r[idx["Time"]]) if r[idx["Time"]] else None, + r[idx["ProductId"]][:50], + r[idx["UserId"]][:50], + int(r[idx["Score"]]) if r[idx["Score"]] else None, + (r[idx["Summary"]] or "")[:500], + r[idx["Text"]] or "", + )) +print(f"[load] parsed {len(rows)} rows") + +# Insert in batches. combined is a computed column so we don't send it. +print("[insert] deleting old rows") +cur.execute("DELETE FROM dbo.reviews;") +conn.commit() + +BATCH = 100 +sql = "INSERT INTO dbo.reviews (Id, Time, ProductId, UserId, Score, Summary, [Text]) VALUES (?, ?, ?, ?, ?, ?, ?)" +start = time.time() +for i in range(0, len(rows), BATCH): + batch = rows[i:i+BATCH] + cur.executemany(sql, batch) + conn.commit() + print(f"[insert] {i+len(batch)}/{len(rows)} (elapsed {time.time()-start:.1f}s)") + +print("[insert] complete") +cur.execute("SELECT COUNT(*) FROM dbo.reviews") +print(f"[verify] row count = {cur.fetchone()[0]}") + +conn.close() +print("[done]") diff --git a/DiskANN/Wikipedia/006-fulltext-setup.sql b/DiskANN/Wikipedia/006-fulltext-setup.sql index 5ae51e3..b3571c3 100644 --- a/DiskANN/Wikipedia/006-fulltext-setup.sql +++ b/DiskANN/Wikipedia/006-fulltext-setup.sql @@ -17,7 +17,9 @@ go select * from sys.fulltext_catalogs go --- Wait ~15 seconds for FT to start and process all the documents, then +-- Full-text population is asynchronous. On Azure SQL Hyperscale for 25000 rows +-- it typically completes in ~30 seconds. If the count below is < 25000, wait a +-- bit longer and re-run just this SELECT. waitfor delay '00:00:15' go diff --git a/DiskANN/Wikipedia/008-semantic-reranking.sql b/DiskANN/Wikipedia/008-semantic-reranking.sql index 437201c..44cf947 100644 --- a/DiskANN/Wikipedia/008-semantic-reranking.sql +++ b/DiskANN/Wikipedia/008-semantic-reranking.sql @@ -1,5 +1,19 @@ /* - Re-Rank results generated in the previous script using CoHere semnatic re-ranker + Re-rank the RRF results from 007-hybrid-search.sql using the Cohere Rerank + model deployed on Azure AI Foundry. + + Prereqs: + - dbo.wikipedia_articles_search_results populated by 007-hybrid-search.sql + - A Cohere-rerank-v4.0-fast (or -pro) deployment on an Azure AI Foundry + AIServices resource. Note the deployment name you chose — Azure deployment + names cannot contain a dot, so a typical deployment name is + 'Cohere-rerank-v4-0-fast'. The 'model' field in the JSON payload below must + match the deployment name, not the raw model name. + + Token budget: the Cohere rerank GlobalStandard SKU frontend enforces a hard + ~1000 tokens/min cap regardless of provisioned capacity. For a live demo, + trim to the top ~5 hybrid results and truncate each document to ~150 chars + (see LEFT([text], 150) below). Full 50-doc payload will hit RateLimitReached. */ -- Uncomment if using SQL Server 2025 --use WikipediaTest @@ -14,18 +28,19 @@ begin end go --- Generate payload for re-ranker, using the result returned by vector search --- Payload formatted as per https://docs.cohere.com/docs/rerank-overview#example-with-structured-data +-- Generate payload for re-ranker, using the top RRF results from hybrid search. +-- Payload format: https://docs.cohere.com/docs/rerank-overview#example-with-structured-data DECLARE @documents JSON = ( - SELECT JSON_ARRAYAGG('Id: ' || id || CHAR(10) || 'Content: ' || [text] RETURNING JSON) FROM wikipedia_articles_search_results -) + SELECT JSON_ARRAYAGG('Id: ' || id || CHAR(10) || 'Content: ' || LEFT([text], 150) RETURNING JSON) + FROM (SELECT TOP 5 * FROM wikipedia_articles_search_results ORDER BY rrf_rank) t +); DECLARE @payload JSON = JSON_OBJECT( - 'model': 'Cohere-rerank-v4.0-fast', + 'model': 'Cohere-rerank-v4-0-fast', -- must match your Foundry deployment name (no dots) 'query': (select q from dbo.wikipedia_search_vectors where id = 1), - 'top_n': 10, + 'top_n': 5, 'documents': @documents -) +); -- Invoke re-ranker model DECLARE @response NVARCHAR(MAX); diff --git a/DiskANN/Wikipedia/009-wikipedia-fp16.sql b/DiskANN/Wikipedia/009-wikipedia-fp16.sql index aa4b1d6..9a1d8c7 100644 --- a/DiskANN/Wikipedia/009-wikipedia-fp16.sql +++ b/DiskANN/Wikipedia/009-wikipedia-fp16.sql @@ -1,175 +1,185 @@ /* - Test the new helf-precision support for vectors - By converting existing single-precision vectors to half-precision vectors - and then do a test run using both to see if there are any differences in the outcome + Test the half-precision (fp16) VECTOR support. + Converts the existing fp32 embeddings to fp16, builds a DiskANN index on + both, and compares storage size + recall of ANN(fp32) vs ANN(fp16) vs KNN(fp32). + + Prereqs (from earlier scripts): + - dbo.wikipedia_articles_embeddings loaded with 25000 rows + - content_vector (VECTOR(1536)) populated + - EXTERNAL MODEL Ada2Embeddings (text-embedding-ada-002) -- the corpus was + embedded with ada-002; querying with a different 1536-dim model gives noise */ -- Uncomment if using SQL Server 2025 -- USE WikipediaTest +-- GO --- Add half-precision vector column -alter table [dbo].[wikipedia_articles_embeddings] -add content_vector_fp16 vector(1536, float16) -go +-- --------------------------------------------------------------------------- +-- Step 1. Add the fp16 column (skip if it already exists) +-- --------------------------------------------------------------------------- +IF COL_LENGTH('dbo.wikipedia_articles_embeddings', 'content_vector_fp16') IS NULL +BEGIN + ALTER TABLE [dbo].[wikipedia_articles_embeddings] + ADD content_vector_fp16 VECTOR(1536, float16); +END +GO --- View the metadata -select - [name] AS column_name, +-- View the metadata: fp32 vs fp16 columns side by side +SELECT + [name] AS column_name, system_type_id, user_type_id, vector_dimensions, vector_base_type, vector_base_type_desc -from - sys.columns -where - object_id = object_id('[dbo].[wikipedia_articles_embeddings]') -go - --- Remove existing vector indexes -select * from sys.vector_indexes -go -drop index if exists vec_idx on [dbo].[wikipedia_articles_embeddings] -drop index if exists vec_idx2 on [dbo].[wikipedia_articles_embeddings] -go -select * from sys.vector_indexes -go - --- Copy the exiting single-precision embeddings to half-precision vector column -update [dbo].[wikipedia_articles_embeddings] ---set content_vector_fp16 = cast(content_vector as vector(1536, float16)) -- Not working at the moment -set content_vector_fp16 = cast(cast(content_vector as json) as vector(1536, float16)) -go - --- View different storage space for single-precision (fp32) vs half-precision (fp16) floating point vector -select - id, title, - DATALENGTH(content_vector) as fp32_bytes, - DATALENGTH(content_vector_fp16) as fp16_bytes -from - [dbo].[wikipedia_articles_embeddings] where title like 'Philosoph%' -go - --- Generate query embeddings -drop table if exists #t; -create table #t (id int, q nvarchar(max), v32 vector(1536, float32), v16 vector(1536, float16)) - -insert into #t (id, q, v32) -select - id, q, ai_generate_embeddings(q use model Ada2Embeddings) -from - (values - (1, N'four legged furry animal'), - (2, N'pink floyd music style') - ) S(id, q) -go -update #t set v16 = cast(cast(v32 as json) as vector(1536, float16)); -select * from #t -go - --- Create vector index of single-precision vectors --- Should take ~30 seconds on a 16 vCore server -create vector index vec_idx32 on [dbo].[wikipedia_articles_embeddings]([content_vector]) -with (metric = 'cosine', type = 'diskann'); -go - --- Create vector index of half-precision vectors --- Should take ~22 seconds on a 16 vCore server -create vector index vec_idx16 on [dbo].[wikipedia_articles_embeddings]([content_vector_fp16]) -with (metric = 'cosine', type = 'diskann'); -go - -select * from sys.vector_indexes -go - -set statistics time on -set statistics io on -go +FROM sys.columns +WHERE object_id = OBJECT_ID('[dbo].[wikipedia_articles_embeddings]') + AND vector_dimensions IS NOT NULL; +GO -/* - RUN KNN (Exact) VECTOR SEARCH -*/ -declare @qv vector(1536, float16) = (select top(1) v16 from #t where id=2); -select top (50) id, vector_distance('cosine', @qv, [content_vector_fp16]) as distance, title -from [dbo].[wikipedia_articles_embeddings] -order by distance; -go +-- --------------------------------------------------------------------------- +-- Step 2. Drop existing vector indexes so we can rebuild against both columns +-- --------------------------------------------------------------------------- +SELECT i.name AS index_name +FROM sys.vector_indexes v +JOIN sys.indexes i ON v.object_id = i.object_id AND v.index_id = i.index_id +WHERE v.object_id = OBJECT_ID('dbo.wikipedia_articles_embeddings'); +GO -/* - RUN ANN (Approximate) VECTOR SEARCH -*/ -declare @qv vector(1536, float16) = (select top(1) v16 from #t where id = 2); -select - t.id, s.distance, t.title -from - vector_search( - table = [dbo].[wikipedia_articles_embeddings] as t, - column = [content_vector_fp16], - similar_to = @qv, - metric = 'cosine', - top_n = 50 - ) as s -order by s.distance, title -; -go +DROP INDEX IF EXISTS vec_idx ON [dbo].[wikipedia_articles_embeddings]; +DROP INDEX IF EXISTS vec_idx2 ON [dbo].[wikipedia_articles_embeddings]; +DROP INDEX IF EXISTS vec_idx32 ON [dbo].[wikipedia_articles_embeddings]; +DROP INDEX IF EXISTS vec_idx16 ON [dbo].[wikipedia_articles_embeddings]; +GO -/* - Calculate Recall and compare fp16 vs fp32 -*/ -declare @n int = 100; -declare @qv32 vector(1536, float32), @qv16 vector(1536, float16); -select top(1) @qv32 = v32, @qv16 = v16 from #t where id = 1; -with cteANN32 as -( - select top (@n) - t.id, s.distance, t.title - from - vector_search( - table = [dbo].[wikipedia_articles_embeddings] as t, - column = [content_vector], - similar_to = @qv32, - metric = 'cosine', - top_n = @n - ) as s - order by s.distance, id +-- --------------------------------------------------------------------------- +-- Step 3. Materialize fp16 from fp32 +-- Note: direct CAST from VECTOR to VECTOR is not supported yet; go via JSON. +-- --------------------------------------------------------------------------- +UPDATE [dbo].[wikipedia_articles_embeddings] +SET content_vector_fp16 = CAST(CAST(content_vector AS JSON) AS VECTOR(1536, float16)) +WHERE content_vector_fp16 IS NULL; +GO + +-- Compare storage: fp16 is half the size of fp32 on disk +SELECT TOP 5 + id, title, + DATALENGTH(content_vector) AS fp32_bytes, + DATALENGTH(content_vector_fp16) AS fp16_bytes +FROM [dbo].[wikipedia_articles_embeddings] +WHERE title LIKE 'Philosoph%'; +GO + +-- --------------------------------------------------------------------------- +-- Step 4. Prepare query vectors (fp32 + fp16 for the same text) +-- Uses Ada2Embeddings because the corpus was embedded with ada-002. +-- --------------------------------------------------------------------------- +DROP TABLE IF EXISTS #t; +CREATE TABLE #t (id INT, q NVARCHAR(MAX), v32 VECTOR(1536, float32), v16 VECTOR(1536, float16)); + +INSERT INTO #t (id, q, v32) +SELECT id, q, AI_GENERATE_EMBEDDINGS(q USE MODEL Ada2Embeddings) +FROM (VALUES + (1, N'four legged furry animal'), + (2, N'pink floyd music style') +) s(id, q); + +UPDATE #t SET v16 = CAST(CAST(v32 AS JSON) AS VECTOR(1536, float16)); + +SELECT id, q, DATALENGTH(v32) AS v32_bytes, DATALENGTH(v16) AS v16_bytes FROM #t; +GO + +-- --------------------------------------------------------------------------- +-- Step 5. Build DiskANN indexes on both columns +-- --------------------------------------------------------------------------- +CREATE VECTOR INDEX vec_idx32 ON [dbo].[wikipedia_articles_embeddings](content_vector) + WITH (METRIC = 'cosine', TYPE = 'diskann'); +GO + +CREATE VECTOR INDEX vec_idx16 ON [dbo].[wikipedia_articles_embeddings](content_vector_fp16) + WITH (METRIC = 'cosine', TYPE = 'diskann'); +GO + +SELECT i.name, JSON_VALUE(v.build_parameters, '$.Version') AS version +FROM sys.vector_indexes v +JOIN sys.indexes i ON v.object_id = i.object_id AND v.index_id = i.index_id +WHERE v.object_id = OBJECT_ID('dbo.wikipedia_articles_embeddings'); +GO + +SET STATISTICS TIME ON; +SET STATISTICS IO ON; +GO + +-- --------------------------------------------------------------------------- +-- Step 6. KNN (exact) on the fp16 column +-- --------------------------------------------------------------------------- +DECLARE @qv VECTOR(1536, float16) = (SELECT TOP(1) v16 FROM #t WHERE id = 2); +SELECT TOP (50) id, VECTOR_DISTANCE('cosine', @qv, content_vector_fp16) AS distance, title +FROM [dbo].[wikipedia_articles_embeddings] +ORDER BY distance; +GO + +-- --------------------------------------------------------------------------- +-- Step 7. ANN (approximate) on the fp16 column +-- New syntax: TOP (N) WITH APPROXIMATE ... ORDER BY distance +-- --------------------------------------------------------------------------- +DECLARE @qv VECTOR(1536, float16) = (SELECT TOP(1) v16 FROM #t WHERE id = 2); +SELECT TOP (50) WITH APPROXIMATE + t.id, s.distance, t.title +FROM VECTOR_SEARCH( + TABLE = [dbo].[wikipedia_articles_embeddings] AS t, + COLUMN = content_vector_fp16, + SIMILAR_TO = @qv, + METRIC = 'cosine' +) AS s +ORDER BY s.distance; +GO + +-- --------------------------------------------------------------------------- +-- Step 8. Recall comparison: KNN(fp32) baseline vs ANN(fp32) vs ANN(fp16) +-- Both fp16 ANN and fp32 ANN measured against the exact fp32 KNN result. +-- --------------------------------------------------------------------------- +DECLARE @n INT = 100; +DECLARE @qv32 VECTOR(1536, float32), @qv16 VECTOR(1536, float16); +SELECT TOP(1) @qv32 = v32, @qv16 = v16 FROM #t WHERE id = 1; + +WITH cteANN32 AS ( + SELECT TOP (@n) WITH APPROXIMATE t.id, s.distance, t.title + FROM VECTOR_SEARCH( + TABLE = [dbo].[wikipedia_articles_embeddings] AS t, + COLUMN = content_vector, + SIMILAR_TO = @qv32, + METRIC = 'cosine' + ) AS s + ORDER BY s.distance ), -cteANN16 as -( - select top (@n) - t.id, s.distance, t.title - from - vector_search( - table = [dbo].[wikipedia_articles_embeddings] as t, - column = [content_vector_fp16], - similar_to = @qv16, - metric = 'cosine', - top_n = @n - ) as s - order by s.distance, id +cteANN16 AS ( + SELECT TOP (@n) WITH APPROXIMATE t.id, s.distance, t.title + FROM VECTOR_SEARCH( + TABLE = [dbo].[wikipedia_articles_embeddings] AS t, + COLUMN = content_vector_fp16, + SIMILAR_TO = @qv16, + METRIC = 'cosine' + ) AS s + ORDER BY s.distance ), -cteKNN32 as -( - select top (@n) id, vector_distance('cosine', @qv32, [content_vector]) as distance, title - from [dbo].[wikipedia_articles_embeddings] - order by distance, id +cteKNN32 AS ( + SELECT TOP (@n) id, VECTOR_DISTANCE('cosine', @qv32, content_vector) AS distance, title + FROM [dbo].[wikipedia_articles_embeddings] + ORDER BY distance, id ) -select - k32.id as id_knn, - a32.id as id_ann_fp32, - a16.id as id_ann_fp16, - k32.distance as distance_knn, - a32.distance as distance_ann_fp32, - a16.distance as distance_ann_fp16, - running_recall_fp32 = cast(cast(count(a32.id) over (order by k32.distance) as float) - / cast(count(k32.id) over (order by k32.distance) as float) as decimal(6,3)), - running_recall_fp16 = cast(cast(count(a16.id) over (order by k32.distance) as float) - / cast(count(k32.id) over (order by k32.distance) as float) as decimal(6,3)) -from - cteKNN32 k32 -left outer join - cteANN32 a32 on k32.id = a32.id -left outer join - cteANN16 a16 on k32.id = a16.id -order by - k32.distance -go - +SELECT + k32.id AS id_knn, + a32.id AS id_ann_fp32, + a16.id AS id_ann_fp16, + k32.distance AS distance_knn, + a32.distance AS distance_ann_fp32, + a16.distance AS distance_ann_fp16, + CAST(CAST(COUNT(a32.id) OVER (ORDER BY k32.distance) AS FLOAT) + / CAST(COUNT(k32.id) OVER (ORDER BY k32.distance) AS FLOAT) AS DECIMAL(6,3)) AS running_recall_fp32, + CAST(CAST(COUNT(a16.id) OVER (ORDER BY k32.distance) AS FLOAT) + / CAST(COUNT(k32.id) OVER (ORDER BY k32.distance) AS FLOAT) AS DECIMAL(6,3)) AS running_recall_fp16 +FROM cteKNN32 k32 +LEFT OUTER JOIN cteANN32 a32 ON k32.id = a32.id +LEFT OUTER JOIN cteANN16 a16 ON k32.id = a16.id +ORDER BY k32.distance; +GO diff --git a/DiskANN/diskann-quickstart-azure-sql-improvements.sql b/DiskANN/diskann-quickstart-azure-sql-improvements.sql index f087eba..d44313f 100644 --- a/DiskANN/diskann-quickstart-azure-sql-improvements.sql +++ b/DiskANN/diskann-quickstart-azure-sql-improvements.sql @@ -34,7 +34,8 @@ GO -- Create external model for embeddings -- Replace and with your deployment details -DROP EXTERNAL MODEL IF EXISTS AIEmbeddings; +IF EXISTS (SELECT 1 FROM sys.external_models WHERE name = 'AIEmbeddings') + DROP EXTERNAL MODEL AIEmbeddings; GO -- Replace MODEL with the choice of your embedding model and LOCATION with the URL from the Azure AI Foundry @@ -330,8 +331,12 @@ GO SELECT OBJECT_NAME(object_id) AS table_name, - approximate_staleness_percent, - last_background_task_succeeded + graph_catchup_pending_percent, + quantized_keys_used_percent, + last_background_task_succeeded, + last_background_task_execution_time, + last_background_task_processed_inserts, + last_background_task_processed_deletes FROM sys.dm_db_vector_indexes WHERE OBJECT_NAME(object_id) = 'Articles'; GO @@ -340,14 +345,14 @@ GO -- -- 1. DML operations (INSERT/UPDATE/DELETE) commit immediately -- 2. Changes are visible in search results IMMEDIATELY (no waiting!) --- 3. approximate_staleness_percent increases briefly after DML +-- 3. graph_catchup_pending_percent tracks how much DML the background task still needs to fold into the DiskANN graph -- 4. Background task processes changes asynchronously --- 5. Staleness decreases as background task completes +-- 5. graph_catchup_pending_percent decreases as background task completes -- 6. last_background_task_processed_* counters increment --- Key insight: Queries work perfectly even when staleness > 0! +-- Key insight: Queries work perfectly even when graph_catchup_pending_percent > 0! -- - Recent changes are visible immediately --- - Staleness only affects ranking optimization, not completeness +-- - Pending catch-up only affects ranking optimization, not completeness -- - Background process keeps the index optimized over time -- ============================================================================ @@ -595,10 +600,9 @@ GO -- - Cost estimates (optimizer compares ANN vs KNN costs) -- You write WITH APPROXIMATE, optimizer decides the execution path! - ============================================================================ +-- ============================================================================ ---Clean up +-- Clean up DROP TABLE IF EXISTS dbo.Articles; GO - diff --git a/DiskANN/diskann-quickstart-azure-sql.sql b/DiskANN/diskann-quickstart-azure-sql.sql index 47b1447..39538c7 100644 --- a/DiskANN/diskann-quickstart-azure-sql.sql +++ b/DiskANN/diskann-quickstart-azure-sql.sql @@ -9,6 +9,8 @@ CREATE TABLE dbo.Articles ); -- Step 2: Insert sample data +-- 10 named rows for storytelling + 90 generated rows. +-- DiskANN requires at least 100 non-null vectors to build the index. INSERT INTO Articles (id, title, content, embedding) VALUES (1, 'Intro to AI', 'This article introduces AI concepts.', '[0.1, 0.2, 0.3, 0.4, 0.5]'), @@ -23,6 +25,25 @@ VALUES (10, 'AI Innovations', 'Latest innovations in AI.', '[0.4, 0.7, 0.2, 0.3, 0.1]'); GO +-- Add 90 more rows with pseudo-random 5-dim vectors to satisfy the 100-row minimum for CREATE VECTOR INDEX +INSERT INTO Articles (id, title, content, embedding) +SELECT + 10 + s.value AS id, + CONCAT(N'Article ', 10 + s.value) AS title, + CONCAT(N'Filler content ', 10 + s.value) AS content, + CAST(CONCAT('[', + FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', + FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', + FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', + FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', + FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), + ']') AS VECTOR(5)) AS embedding +FROM GENERATE_SERIES(1, 90) AS s; +GO + +SELECT COUNT(*) AS row_count FROM dbo.Articles; +GO + -- Step 3: Create a vector index on the embedding column CREATE VECTOR INDEX vec_idx ON Articles(embedding) WITH (METRIC = 'Cosine', TYPE = 'DiskANN') @@ -31,7 +52,7 @@ GO -- Step 4: Perform a vector similarity search DECLARE @qv VECTOR(5) = (SELECT TOP(1) embedding FROM Articles WHERE id = 1); -SELECT +SELECT TOP (3) WITH APPROXIMATE t.id, t.title, t.content, @@ -41,36 +62,26 @@ FROM TABLE = Articles AS t, COLUMN = embedding, SIMILAR_TO = @qv, - METRIC = 'Cosine', - TOP_N = 3 + METRIC = 'Cosine' ) AS s -ORDER BY s.distance, t.title; +ORDER BY s.distance; GO -- Step 5: View index details SELECT index_id, [type], [type_desc], vector_index_type, distance_metric, build_parameters FROM sys.vector_indexes WHERE [name] = 'vec_idx'; GO --- Step 6a: Data modification is disabled when DiskANN exist on a table -INSERT INTO Articles (id, title, content, embedding) -VALUES -(11, 'Vectors and Embeddings', 'Everything about vectors and embeddings.', '[0.1, 0.2, 0.3, 0.4, 0.6]'); -GO - --- Step 6b: Allow index to go stale -ALTER DATABASE SCOPED CONFIGURATION -SET ALLOW_STALE_VECTOR_INDEX = ON -GO - --- Step 6c: Data modification is now works +-- Step 6: DML works on a live vector index (current Azure SQL / SQL Server 2025 behavior). +-- INSERT / UPDATE / DELETE are visible in vector search results immediately; +-- the DiskANN graph is maintained asynchronously in the background. INSERT INTO Articles (id, title, content, embedding) VALUES -(11, 'Vectors and Embeddings', 'Everything about vectors and embeddings.', '[0.1, 0.2, 0.3, 0.4, 0.6]'); +(200, 'Vectors and Embeddings', 'Everything about vectors and embeddings.', '[0.1, 0.2, 0.3, 0.4, 0.6]'); GO --- Step 7: Perform a vector similarity search, new data not visible +-- Step 7: The new row shows up in vector search immediately DECLARE @qv VECTOR(5) = (SELECT TOP(1) embedding FROM Articles WHERE id = 1); -SELECT +SELECT TOP (3) WITH APPROXIMATE t.id, t.title, t.content, @@ -80,36 +91,23 @@ FROM TABLE = Articles AS t, COLUMN = embedding, SIMILAR_TO = @qv, - METRIC = 'Cosine', - TOP_N = 3 + METRIC = 'Cosine' ) AS s -ORDER BY s.distance, t.title; -GO - --- Step 8: Re-Create a vector index on the embedding column -DROP INDEX vec_idx ON Articles; -CREATE VECTOR INDEX vec_idx ON Articles(embedding) -WITH (METRIC = 'Cosine', TYPE = 'DiskANN') -ON [PRIMARY]; +ORDER BY s.distance; GO --- Step 9: Data now visible -DECLARE @qv VECTOR(5) = (SELECT TOP(1) embedding FROM Articles WHERE id = 1); +-- Step 8: Observe background maintenance state SELECT - t.id, - t.title, - t.content, - s.distance -FROM - VECTOR_SEARCH( - TABLE = Articles AS t, - COLUMN = embedding, - SIMILAR_TO = @qv, - METRIC = 'Cosine', - TOP_N = 3 - ) AS s -ORDER BY s.distance; + OBJECT_NAME(object_id) AS table_name, + graph_catchup_pending_percent, + last_background_task_succeeded, + last_background_task_execution_time, + last_background_task_processed_inserts, + last_background_task_processed_deletes +FROM sys.dm_db_vector_indexes +WHERE OBJECT_NAME(object_id) = 'Articles'; GO --- Step 6: Clean up by dropping the table -DROP INDEX vec_idx ON Articles; \ No newline at end of file +-- Step 9: Clean up +DROP INDEX vec_idx ON Articles; +DROP TABLE IF EXISTS dbo.Articles; \ No newline at end of file diff --git a/DiskANN/diskann-quickstart-sql-server-2025.sql b/DiskANN/diskann-quickstart-sql-server-2025.sql index 20ed3b7..ec90345 100644 --- a/DiskANN/diskann-quickstart-sql-server-2025.sql +++ b/DiskANN/diskann-quickstart-sql-server-2025.sql @@ -22,6 +22,8 @@ CREATE TABLE dbo.Articles ); -- Step 2: Insert sample data +-- 10 named rows for storytelling + 90 generated rows. +-- DiskANN requires at least 100 non-null vectors to build the index. INSERT INTO Articles (id, title, content, embedding) VALUES (1, 'Intro to AI', 'This article introduces AI concepts.', '[0.1, 0.2, 0.3, 0.4, 0.5]'), @@ -36,6 +38,25 @@ VALUES (10, 'AI Innovations', 'Latest innovations in AI.', '[0.4, 0.7, 0.2, 0.3, 0.1]'); GO +-- Add 90 more rows with pseudo-random 5-dim vectors to satisfy the 100-row minimum for CREATE VECTOR INDEX +INSERT INTO Articles (id, title, content, embedding) +SELECT + 10 + s.value AS id, + CONCAT(N'Article ', 10 + s.value) AS title, + CONCAT(N'Filler content ', 10 + s.value) AS content, + CAST(CONCAT('[', + FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', + FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', + FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', + FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', + FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), + ']') AS VECTOR(5)) AS embedding +FROM GENERATE_SERIES(1, 90) AS s; +GO + +SELECT COUNT(*) AS row_count FROM dbo.Articles; +GO + -- Step 3: Create a vector index on the embedding column CREATE VECTOR INDEX vec_idx ON Articles(embedding) WITH (METRIC = 'Cosine', TYPE = 'DiskANN') @@ -44,7 +65,7 @@ GO -- Step 4: Perform a vector similarity search DECLARE @qv VECTOR(5) = (SELECT TOP(1) embedding FROM Articles WHERE id = 1); -SELECT +SELECT TOP (3) WITH APPROXIMATE t.id, t.title, t.content, @@ -54,10 +75,9 @@ FROM TABLE = Articles AS t, COLUMN = embedding, SIMILAR_TO = @qv, - METRIC = 'Cosine', - TOP_N = 3 + METRIC = 'Cosine' ) AS s -ORDER BY s.distance, t.title; +ORDER BY s.distance; GO -- Step 5: View index details diff --git a/Hybrid-Search/hybrid_search.py b/Hybrid-Search/hybrid_search.py index 9df9d57..6c3396c 100644 --- a/Hybrid-Search/hybrid_search.py +++ b/Hybrid-Search/hybrid_search.py @@ -35,8 +35,11 @@ cursor = conn.cursor() for id, (content, embedding) in enumerate(zip(sentences, embeddings)): + # pyodbc binds strings as ntext by default; CAST via NVARCHAR(MAX) first + # so SQL can then coerce the JSON array into VECTOR(384). cursor.execute(f""" - INSERT INTO dbo.documents (id, content, embedding) VALUES (?, ?, CAST(? AS VECTOR(384))); + INSERT INTO dbo.documents (id, content, embedding) + VALUES (?, ?, CAST(CAST(? AS NVARCHAR(MAX)) AS VECTOR(384))); """, id, content, @@ -60,7 +63,7 @@ results = cursor.execute(f""" DECLARE @k INT = ?; DECLARE @q NVARCHAR(1000) = ?; - DECLARE @v VECTOR(384) = CAST(? AS VECTOR(384)); + DECLARE @v VECTOR(384) = CAST(CAST(? AS NVARCHAR(MAX)) AS VECTOR(384)); WITH keyword_search AS ( SELECT TOP(@k) id, diff --git a/RAG-with-Documents/.env.sample b/RAG-with-Documents/.env.sample index 1ca981a..3c2af49 100644 --- a/RAG-with-Documents/.env.sample +++ b/RAG-with-Documents/.env.sample @@ -7,8 +7,8 @@ AZUREDOCINTELLIGENCE_ENDPOINT = "https://.cogn AZUREDOCINTELLIGENCE_API_KEY = "" # Use only one of the below. The one you are not using should be commented out. -# For Entra ID Service Principle Authentication -ENTRA_CONNECTION_STRING="Driver={ODBC Driver 18 for SQL Server};LongAsMax=yes;Server=tcp:.database.windows.net;Database=;" +# For Entra ID passwordless authentication (recommended) +ENTRA_CONNECTION_STRING="Server=tcp:.database.windows.net;Database=;" # For SQL Authentication -SQL_CONNECTION_STRING="Driver={ODBC Driver 18 for SQL Server};LongAsMax=yes;Server=tcp:.database.windows.net;Database=;Uid=;Pwd=;" +SQL_CONNECTION_STRING="Server=tcp:.database.windows.net;Database=;Uid=;Pwd=;" diff --git a/RAG-with-Documents/RAG-with-resumes.ipynb b/RAG-with-Documents/RAG-with-resumes.ipynb index 1b4dc0c..7aafbc6 100644 --- a/RAG-with-Documents/RAG-with-resumes.ipynb +++ b/RAG-with-Documents/RAG-with-resumes.ipynb @@ -1,28 +1,12 @@ { - "metadata": { - "kernelspec": { - "name": "python3", - "display_name": "Python 3", - "language": "python" - }, - "language_info": { - "name": "python", - "version": "3.11.9", - "mimetype": "text/x-python", - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "pygments_lexer": "ipython3", - "nbconvert_exporter": "python", - "file_extension": ".py" - } - }, - "nbformat_minor": 2, - "nbformat": 4, "cells": [ { + "attachments": {}, "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "507219d1-d713-4c41-86d5-e938bf69627c", + "language": "sql" + }, "source": [ "# Leveraging Azure SQL DB’s Native Vector Capabilities for Enhanced Resume Matching with Azure Document Intelligence and RAG\n", "\n", @@ -54,7 +38,7 @@ "- **Azure OpenAI Access**: Apply for access in the desired Azure subscription at [https://aka.ms/oai/access](https://aka.ms/oai/access)\n", "- **Azure OpenAI Resource**: Deploy an embeddings model (e.g., `text-embedding-small` or `text-embedding-ada-002`) and a `GPT-4.0` model for chat completion. Refer to the [resource deployment guide](https://learn.microsoft.com/azure/ai-services/openai/how-to/create-resource)\n", "- **Python**: Version 3.7.1 or later from Python.org. (Sample has been tested with Python 3.11)\n", - "- **Python Libraries**: Install the required libraries openai, num2words, matplotlib, plotly, scipy, scikit-learn, pandas, tiktoken, and pyodbc.\n", + "- **Python Libraries**: Install the required libraries openai, num2words, matplotlib, plotly, scipy, scikit-learn, pandas, tiktoken, and mssql-python.\n", "- **Jupyter Notebooks**: Use within [Azure Data Studio](https://learn.microsoft.com/en-us/azure-data-studio/notebooks/notebooks-guidance) or Visual Studio Code .\n", "\n", "Code snippets are adapted from the [Azure OpenAI Service embeddings Tutorial](https://learn.microsoft.com/en-us/azure/ai-services/openai/tutorials/embeddings?tabs=python-new%2Ccommand-line&pivots=programming-language-python)\n", @@ -80,34 +64,25 @@ "## Running the Notebook\n", "\n", "To [execute the notebook](https://learn.microsoft.com/azure-data-studio/notebooks/notebooks-python-kernel), connect to your Azure SQL database using Azure Data Studio, which can be downloaded [here](https://azure.microsoft.com/products/data-studio)" - ], - "metadata": { - "azdata_cell_guid": "507219d1-d713-4c41-86d5-e938bf69627c", - "language": "sql" - }, - "attachments": {} + ] }, { "cell_type": "code", - "source": [ - "#Setup the python libraries required for this notebook\n", - "#Please ensure that you navigate to the directory containing the `requirements.txt` file in your terminal\n", - "%pip install -r requirements.txt" - ], + "execution_count": null, "metadata": { "azdata_cell_guid": "fe37c601-5918-4055-badc-6c0ba90c68ce", "language": "python" }, "outputs": [], - "execution_count": null + "source": [ + "#Setup the python libraries required for this notebook\n", + "#Please ensure that you navigate to the directory containing the `requirements.txt` file in your terminal\n", + "%pip install -r requirements.txt" + ] }, { "cell_type": "code", - "source": [ - "#Load the env details\n", - "from dotenv import load_dotenv\n", - "load_dotenv()" - ], + "execution_count": 3, "metadata": { "azdata_cell_guid": "4c29709e-1c3a-495d-83ec-05737e220847", "language": "python" @@ -115,32 +90,44 @@ "outputs": [ { "data": { - "text/plain": "True" + "text/plain": [ + "True" + ] }, - "metadata": {}, "execution_count": 3, + "metadata": {}, "output_type": "execute_result" } ], - "execution_count": 3 + "source": [ + "#Load the env details\n", + "from dotenv import load_dotenv\n", + "load_dotenv()" + ] }, { + "attachments": {}, "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "4b543f05-9036-4887-8737-09aa9f865ec2", + "language": "python" + }, "source": [ "# **PART 1: Extracting and Chunking Text from PDF Resumes using Azure Document Intelligence**\n", "\n", "Create an instance of the [DocumentAnalysisClient](https://learn.microsoft.com/azure/ai-services/document-intelligence/create-document-intelligence-resource?view=doc-intel-4.0.0#get-endpoint-url-and-keys) using the endpoint and API key. \n", "\n", "[Azure Document Intelligence](https://learn.microsoft.com/azure/ai-services/document-intelligence/?view=doc-intel-4.0.0_)(previously known as Form Recognizer) is a Azure cloud service that uses machine learning to analyze text and structured data from your documents. This client will be used to send requests to the [Azure Document Intelligence](https://learn.microsoft.com/python/api/overview/azure/ai-formrecognizer-readme?view=azure-python) service and receive responses containing the extracted text from the PDF resumes." - ], - "metadata": { - "azdata_cell_guid": "4b543f05-9036-4887-8737-09aa9f865ec2", - "language": "python" - }, - "attachments": {} + ] }, { "cell_type": "code", + "execution_count": 4, + "metadata": { + "azdata_cell_guid": "b20cc66a-50ce-4486-b275-d4683f4ba545", + "language": "python" + }, + "outputs": [], "source": [ "import os\n", "import re\n", @@ -156,16 +143,15 @@ " endpoint=endpoint,\n", " credential=AzureKeyCredential(api_key)\n", ")\n" - ], - "metadata": { - "azdata_cell_guid": "b20cc66a-50ce-4486-b275-d4683f4ba545", - "language": "python" - }, - "outputs": [], - "execution_count": 4 + ] }, { + "attachments": {}, "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "1ecc04b0-1a5f-4d48-819d-2cc06a070062", + "language": "python" + }, "source": [ "### **Analyze input documents using prebuilt model in Azure Document Intelligence**\n", "\n", @@ -177,15 +163,118 @@ "- When faced with content that exceeds the embedding limit, we usually also chunk the content into smaller pieces and then embed those one at a time. Here we will use [tiktoken](https://github.com/openai/tiktoken?tab=readme-ov-file) to chunk the extracted text into token sizes of 500, as we will later pass the extracted chunks to to the `text-embedding-small` model for [generating text embeddings](https://learn.microsoft.com/azure/ai-services/openai/tutorials/embeddings?tabs=python-new%2Ccommand-line&pivots=programming-language-python) as this has a model input token limit of 8192.\n", "\n", "**Note**: You need to provide the location of the folder where the PDF files reside in the below script." - ], - "metadata": { - "azdata_cell_guid": "1ecc04b0-1a5f-4d48-819d-2cc06a070062", - "language": "python" - }, - "attachments": {} + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "0355f92c-0546-4eac-aea8-b55d8bbef194", + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of PDF files in the directory: 120\n", + "Processing file 1/120: 92069209.pdf\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of chunks for file 92069209.pdf: 3\n", + "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 0_0, Chunk Length: 3035, Chunk Text: INFORMATION TECHNOLOGY TECHNICIAN I Summary Versat...\n", + "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 0_1, Chunk Length: 2959, Chunk Text: Disaster Recovery plan and procedures Researching...\n", + "File: 92069209.pdf, Chunk ID: 2, Unique Chunk ID: 0_2, Chunk Length: 2048, Chunk Text: Installing configuring and supporting McAfee antiv...\n", + "Processing file 2/120: 92069209.pdf\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of chunks for file 92069209.pdf: 3\n", + "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 1_0, Chunk Length: 3191, Chunk Text: INFORMATION TECHNOLOGY MANAGER Professional Summar...\n", + "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 1_1, Chunk Length: 2744, Chunk Text: network which entailed changing software and LAN c...\n", + "File: 92069209.pdf, Chunk ID: 2, Unique Chunk ID: 1_2, Chunk Length: 729, Chunk Text: i4 City State 2015 Master of Science Information ...\n", + "Processing file 3/120: 92069209.pdf\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of chunks for file 92069209.pdf: 2\n", + "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 2_0, Chunk Length: 3024, Chunk Text: WORKING RF SYSTEMS ENGINEER Qualifications Microso...\n", + "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 2_1, Chunk Length: 1892, Chunk Text: surveys ElectricalValidation Engineer May 2011 to ...\n", + "Processing file 4/120: 92069209.pdf\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of chunks for file 92069209.pdf: 2\n", + "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 3_0, Chunk Length: 2934, Chunk Text: INFORMATION TECHNOLOGY MANAGER Summary Dedicated I...\n", + "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 3_1, Chunk Length: 1917, Chunk Text: XP Vista and Mac operating systems Responsible fo...\n", + "Processing file 5/120: 92069209.pdf\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of chunks for file 92069209.pdf: 2\n", + "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 4_0, Chunk Length: 2854, Chunk Text: IT MANAGEMENT Career Overview Detailoriented profe...\n", + "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 4_1, Chunk Length: 2209, Chunk Text: to the device itself being recharged by a small so...\n", + "Processing file 6/120: 92069209.pdf\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of chunks for file 92069209.pdf: 2\n", + "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 5_0, Chunk Length: 2928, Chunk Text: INFORMATION TECHNOLOGY SPECIALIST Professional Sum...\n", + "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 5_1, Chunk Length: 1336, Chunk Text: business and or technical problems Applies metrics...\n", + "Processing file 7/120: 92069209.pdf\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of chunks for file 92069209.pdf: 3\n", + "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 6_0, Chunk Length: 3295, Chunk Text: BRANCH CHIEF INFORMATION TECHNOLOGY SPECIALIST Pro...\n", + "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 6_1, Chunk Length: 3179, Chunk Text: computer network operations plans including defens...\n", + "File: 92069209.pdf, Chunk ID: 2, Unique Chunk ID: 6_2, Chunk Length: 681, Chunk Text: 2009 DISA Action Officers Course 10Dec2009 DOD Inf...\n", + "Processing file 8/120: 92069209.pdf\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of chunks for file 92069209.pdf: 2\n", + "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 7_0, Chunk Length: 2784, Chunk Text: INFORMATION TECHNOLOGY COORDINATOR Career Overview...\n", + "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 7_1, Chunk Length: 2741, Chunk Text: dispatch process to an automated delivery system v...\n", + "Processing file 9/120: 92069209.pdf\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of chunks for file 92069209.pdf: 2\n", + "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 8_0, Chunk Length: 3013, Chunk Text: MANAGER INFORMATION TECHNOLOGY AND BUILDING AUTOM...\n", + "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 8_1, Chunk Length: 1972, Chunk Text: and Operation New Dawn focusing on network securit...\n", + "Processing file 10/120: 92069209.pdf\n" + ] + } + ], "source": [ "import os\n", "import re\n", @@ -265,74 +354,12 @@ "\n", "df = pd.DataFrame(data)\n", "df.head(3)\n", - "\n", - "" - ], - "metadata": { - "azdata_cell_guid": "0355f92c-0546-4eac-aea8-b55d8bbef194", - "language": "python" - }, - "outputs": [ - { - "name": "stdout", - "text": "Number of PDF files in the directory: 120\nProcessing file 1/120: 92069209.pdf\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Number of chunks for file 92069209.pdf: 3\nFile: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 0_0, Chunk Length: 3035, Chunk Text: INFORMATION TECHNOLOGY TECHNICIAN I Summary Versat...\nFile: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 0_1, Chunk Length: 2959, Chunk Text: Disaster Recovery plan and procedures Researching...\nFile: 92069209.pdf, Chunk ID: 2, Unique Chunk ID: 0_2, Chunk Length: 2048, Chunk Text: Installing configuring and supporting McAfee antiv...\nProcessing file 2/120: 92069209.pdf\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Number of chunks for file 92069209.pdf: 3\nFile: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 1_0, Chunk Length: 3191, Chunk Text: INFORMATION TECHNOLOGY MANAGER Professional Summar...\nFile: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 1_1, Chunk Length: 2744, Chunk Text: network which entailed changing software and LAN c...\nFile: 92069209.pdf, Chunk ID: 2, Unique Chunk ID: 1_2, Chunk Length: 729, Chunk Text: i4 City State 2015 Master of Science Information ...\nProcessing file 3/120: 92069209.pdf\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Number of chunks for file 92069209.pdf: 2\nFile: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 2_0, Chunk Length: 3024, Chunk Text: WORKING RF SYSTEMS ENGINEER Qualifications Microso...\nFile: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 2_1, Chunk Length: 1892, Chunk Text: surveys ElectricalValidation Engineer May 2011 to ...\nProcessing file 4/120: 92069209.pdf\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Number of chunks for file 92069209.pdf: 2\nFile: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 3_0, Chunk Length: 2934, Chunk Text: INFORMATION TECHNOLOGY MANAGER Summary Dedicated I...\nFile: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 3_1, Chunk Length: 1917, Chunk Text: XP Vista and Mac operating systems Responsible fo...\nProcessing file 5/120: 92069209.pdf\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Number of chunks for file 92069209.pdf: 2\nFile: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 4_0, Chunk Length: 2854, Chunk Text: IT MANAGEMENT Career Overview Detailoriented profe...\nFile: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 4_1, Chunk Length: 2209, Chunk Text: to the device itself being recharged by a small so...\nProcessing file 6/120: 92069209.pdf\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Number of chunks for file 92069209.pdf: 2\nFile: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 5_0, Chunk Length: 2928, Chunk Text: INFORMATION TECHNOLOGY SPECIALIST Professional Sum...\nFile: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 5_1, Chunk Length: 1336, Chunk Text: business and or technical problems Applies metrics...\nProcessing file 7/120: 92069209.pdf\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Number of chunks for file 92069209.pdf: 3\nFile: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 6_0, Chunk Length: 3295, Chunk Text: BRANCH CHIEF INFORMATION TECHNOLOGY SPECIALIST Pro...\nFile: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 6_1, Chunk Length: 3179, Chunk Text: computer network operations plans including defens...\nFile: 92069209.pdf, Chunk ID: 2, Unique Chunk ID: 6_2, Chunk Length: 681, Chunk Text: 2009 DISA Action Officers Course 10Dec2009 DOD Inf...\nProcessing file 8/120: 92069209.pdf\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Number of chunks for file 92069209.pdf: 2\nFile: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 7_0, Chunk Length: 2784, Chunk Text: INFORMATION TECHNOLOGY COORDINATOR Career Overview...\nFile: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 7_1, Chunk Length: 2741, Chunk Text: dispatch process to an automated delivery system v...\nProcessing file 9/120: 92069209.pdf\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Number of chunks for file 92069209.pdf: 2\nFile: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 8_0, Chunk Length: 3013, Chunk Text: MANAGER INFORMATION TECHNOLOGY AND BUILDING AUTOM...\nFile: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 8_1, Chunk Length: 1972, Chunk Text: and Operation New Dawn focusing on network securit...\nProcessing file 10/120: 92069209.pdf\n", - "output_type": "stream" - } - ], - "execution_count": null + "\n" + ] }, { "cell_type": "code", - "source": [ - "#read the top5 rows of the dataframe\r\n", - "df.head(5)\r\n", - "" - ], + "execution_count": 54, "metadata": { "azdata_cell_guid": "3c0b7dba-c798-40d1-ac92-80288633497a", "language": "python" @@ -340,40 +367,121 @@ "outputs": [ { "data": { - "text/plain": " file_name chunk_id chunk_text \\\n0 10089434.pdf 0 INFORMATION TECHNOLOGY TECHNICIAN I Summary Ve... \n1 10089434.pdf 1 Disaster Recovery plan and procedures Researc... \n2 10089434.pdf 2 Installing configuring and supporting McAfee a... \n3 10247517.pdf 0 INFORMATION TECHNOLOGY MANAGER Professional Su... \n4 10247517.pdf 1 network which entailed changing software and L... \n\n unique_chunk_id chunk_length \n0 0_0 3035 \n1 0_1 2959 \n2 0_2 2048 \n3 1_0 3191 \n4 1_1 2744 ", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
file_namechunk_idchunk_textunique_chunk_idchunk_length
010089434.pdf0INFORMATION TECHNOLOGY TECHNICIAN I Summary Ve...0_03035
110089434.pdf1Disaster Recovery plan and procedures Researc...0_12959
210089434.pdf2Installing configuring and supporting McAfee a...0_22048
310247517.pdf0INFORMATION TECHNOLOGY MANAGER Professional Su...1_03191
410247517.pdf1network which entailed changing software and L...1_12744
\n
" + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
file_namechunk_idchunk_textunique_chunk_idchunk_length
010089434.pdf0INFORMATION TECHNOLOGY TECHNICIAN I Summary Ve...0_03035
110089434.pdf1Disaster Recovery plan and procedures Researc...0_12959
210089434.pdf2Installing configuring and supporting McAfee a...0_22048
310247517.pdf0INFORMATION TECHNOLOGY MANAGER Professional Su...1_03191
410247517.pdf1network which entailed changing software and L...1_12744
\n", + "
" + ], + "text/plain": [ + " file_name chunk_id chunk_text \\\n", + "0 10089434.pdf 0 INFORMATION TECHNOLOGY TECHNICIAN I Summary Ve... \n", + "1 10089434.pdf 1 Disaster Recovery plan and procedures Researc... \n", + "2 10089434.pdf 2 Installing configuring and supporting McAfee a... \n", + "3 10247517.pdf 0 INFORMATION TECHNOLOGY MANAGER Professional Su... \n", + "4 10247517.pdf 1 network which entailed changing software and L... \n", + "\n", + " unique_chunk_id chunk_length \n", + "0 0_0 3035 \n", + "1 0_1 2959 \n", + "2 0_2 2048 \n", + "3 1_0 3191 \n", + "4 1_1 2744 " + ] }, - "metadata": {}, "execution_count": 54, + "metadata": {}, "output_type": "execute_result" } ], - "execution_count": 54 + "source": [ + "#read the top5 rows of the dataframe\n", + "df.head(5)\n" + ] }, { + "attachments": {}, "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "655a8389-1018-4e80-a39d-85187cf3c46f", + "language": "python" + }, "source": [ "### **Tokenization vs. Character Length (OPTIONAL)**\n", "\n", "In this section, we will explore the difference between the character length of a text chunk and its tokenized representation. Character length simply counts the number of characters in a text, while tokenization breaks the text into meaningful units called tokens.\n", "\n", "Character Length First, let’s add a new column to our DataFrame to view the length of each chunk in terms of characters: Here, chunk\\_length represents the number of characters in each chunk." - ], - "metadata": { - "azdata_cell_guid": "655a8389-1018-4e80-a39d-85187cf3c46f", - "language": "python" - }, - "attachments": {} + ] }, { "cell_type": "code", - "source": [ - "# Add a new column 'chunk_length' to the DataFrame to view the length of each chunk\n", - "df['chunk_length'] = df['chunk_text'].apply(len)\n", - "\n", - "# Display the first few rows of the DataFrame with the new column\n", - "print(df[['file_name', 'chunk_id', 'chunk_length']].head(5))\n" - ], + "execution_count": 55, "metadata": { "azdata_cell_guid": "5e171928-764a-4d61-899a-83a8e0c3ac79", "language": "python" @@ -381,14 +489,32 @@ "outputs": [ { "name": "stdout", - "text": " file_name chunk_id chunk_length\n0 10089434.pdf 0 3035\n1 10089434.pdf 1 2959\n2 10089434.pdf 2 2048\n3 10247517.pdf 0 3191\n4 10247517.pdf 1 2744\n", - "output_type": "stream" + "output_type": "stream", + "text": [ + " file_name chunk_id chunk_length\n", + "0 10089434.pdf 0 3035\n", + "1 10089434.pdf 1 2959\n", + "2 10089434.pdf 2 2048\n", + "3 10247517.pdf 0 3191\n", + "4 10247517.pdf 1 2744\n" + ] } ], - "execution_count": 55 + "source": [ + "# Add a new column 'chunk_length' to the DataFrame to view the length of each chunk\n", + "df['chunk_length'] = df['chunk_text'].apply(len)\n", + "\n", + "# Display the first few rows of the DataFrame with the new column\n", + "print(df[['file_name', 'chunk_id', 'chunk_length']].head(5))\n" + ] }, { + "attachments": {}, "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "9b87f84b-149c-4eb5-b9f0-c1b55f6d606c", + "language": "python" + }, "source": [ "### Tokenization\n", "To understand how text ultimately is tokenized, it can be helpful to run the below code: \n", @@ -398,22 +524,11 @@ "- If you then check the length of the decode variable, you'll find it matches 500 our specified token number. It is simply a way of making sure none of the data we pass to the model for tokenization and embedding exceeds the input token limit of 8,192\n", "\n", "- When we pass the documents to the embeddings model, it will break the documents into tokens similar (though not necessarily identical) to the examples below and then convert the tokens to a series of floating point numbers that will be accessible via vector search" - ], - "metadata": { - "azdata_cell_guid": "9b87f84b-149c-4eb5-b9f0-c1b55f6d606c", - "language": "python" - }, - "attachments": {} + ] }, { "cell_type": "code", - "source": [ - "import tiktoken\n", - "tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n", - "sample_encode = tokenizer.encode(df.chunk_text[0]) \n", - "decode = tokenizer.decode_tokens_bytes(sample_encode)\n", - "decode\n" - ], + "execution_count": 14, "metadata": { "azdata_cell_guid": "28188280-2bef-414a-a663-a017c944bc19", "language": "python" @@ -421,20 +536,525 @@ "outputs": [ { "data": { - "text/plain": "[b'IN',\n b'FORMATION',\n b' TECHNO',\n b'LOGY',\n b' TECH',\n b'NIC',\n b'IAN',\n b' I',\n b' Summary',\n b' Vers',\n b'atile',\n b' Systems',\n b' Administrator',\n b' possessing',\n b' superior',\n b' troubleshooting',\n b' skills',\n b' for',\n b' networking',\n b' issues',\n b' end',\n b' user',\n b' problems',\n b' and',\n b' network',\n b' security',\n b' Experienced',\n b' in',\n b' server',\n b' management',\n b' systems',\n b' analysis',\n b' and',\n b' offering',\n b' inde',\n b'pth',\n b' understanding',\n b' of',\n b' IT',\n b' infrastructure',\n b' areas',\n b' Detail',\n b'oriented',\n b' independent',\n b' and',\n b' focused',\n b' on',\n b' taking',\n b' a',\n b' systematic',\n b' approach',\n b' to',\n b' solving',\n b' complex',\n b' problems',\n b' Demonstr',\n b'ated',\n b' exceptional',\n b' technical',\n b' knowledge',\n b' and',\n b' skills',\n b' while',\n b' working',\n b' with',\n b' various',\n b' teams',\n b' to',\n b' achieve',\n b' shared',\n b' goals',\n b' and',\n b' objectives',\n b' Highlights',\n b' ',\n b' Active',\n b' Directory',\n b' ',\n b' New',\n b' technology',\n b' and',\n b' product',\n b' research',\n b' ',\n b' Group',\n b' Policy',\n b' Objects',\n b' ',\n b' Office',\n b' ',\n b'365',\n b' and',\n b' Azure',\n b' ',\n b' PowerShell',\n b' and',\n b' VB',\n b'Script',\n b' ',\n b' Storage',\n b' management',\n b' ',\n b' Microsoft',\n b' Exchange',\n b' ',\n b' Enterprise',\n b' backup',\n b' management',\n b' ',\n b' VM',\n b'Ware',\n b' experience',\n b' ',\n b' Disaster',\n b' recovery',\n b' Experience',\n b' Information',\n b' Technology',\n b' Technician',\n b' I',\n b' Aug',\n b' ',\n b'200',\n b'7',\n b' to',\n b' Current',\n b' Company',\n b' Name',\n b' i',\n b'14',\n b' City',\n b' State',\n b' ',\n b' M',\n b'igr',\n b'ating',\n b' and',\n b' managing',\n b' user',\n b' accounts',\n b' in',\n b' Microsoft',\n b' Office',\n b' ',\n b'365',\n b' and',\n b' Exchange',\n b' Online',\n b' ',\n b' Creating',\n b' and',\n b' managing',\n b' virtual',\n b' machines',\n b' for',\n b' systems',\n b' such',\n b' as',\n b' domain',\n b' controllers',\n b' and',\n b' Active',\n b' Directory',\n b' Federation',\n b' Services',\n b' A',\n b'DFS',\n b' in',\n b' Microsoft',\n b' Windows',\n b' Azure',\n b' I',\n b'aaS',\n b' ',\n b' Creating',\n b' and',\n b' managing',\n b' storage',\n b' in',\n b' Microsoft',\n b' Windows',\n b' Azure',\n b' I',\n b'aaS',\n b' ',\n b' Installing',\n b' and',\n b' configuring',\n b' St',\n b'or',\n b'Simple',\n b' i',\n b'SC',\n b'SI',\n b' cloud',\n b' array',\n b' ST',\n b'aa',\n b'SB',\n b'aaS',\n b' ',\n b' Installing',\n b' configuring',\n b' and',\n b' testing',\n b' Twin',\n b'str',\n b'ata',\n b' i',\n b'SC',\n b'SI',\n b' cloud',\n b' array',\n b' ST',\n b'aa',\n b'SB',\n b'aaS',\n b' ',\n b' Collabor',\n b'ating',\n b' on',\n b' project',\n b' plan',\n b' for',\n b' Office',\n b' ',\n b'365',\n b' migration',\n b' ',\n b' Developing',\n b' detailed',\n b' specifications',\n b' for',\n b' the',\n b' Office',\n b' ',\n b'365',\n b' migration',\n b' including',\n b' business',\n b'case',\n b' documentation',\n b' cost',\n b' benefit',\n b' analyses',\n b' technical',\n b' diagrams',\n b' and',\n b' work',\n b' flow',\n b' documentation',\n b' ',\n b' Received',\n b' training',\n b' in',\n b' MVC',\n b' ',\n b'4',\n b' for',\n b' Visual',\n b' Studio',\n b' using',\n b' ',\n b' Net',\n b' Framework',\n b' ',\n b'445',\n b' to',\n b' develop',\n b' application',\n b' using',\n b' HTML',\n b'5',\n b' and',\n b' CSS',\n b'3',\n b' ',\n b' Installing',\n b' configuring',\n b' and',\n b' supporting',\n b' Linux',\n b' machines',\n b' for',\n b' the',\n b' open',\n b' WiFi',\n b' network',\n b' project',\n b' ',\n b' Comp',\n b'iling',\n b' and',\n b' generating',\n b' statistical',\n b' information',\n b' concerning',\n b' wireless',\n b' network',\n b' traffic',\n b' using',\n b' C',\n b'act',\n b'i',\n b' ',\n b' Config',\n b'uring',\n b' wireless',\n b' LAN',\n b' router',\n b' networking',\n b' and',\n b' security',\n b' access',\n b' ',\n b' Installing',\n b' and',\n b' configuring',\n b' wireless',\n b' certificates',\n b' ',\n b' Developing',\n b' detailed',\n b' specifications',\n b' for',\n b' the',\n b' acquisition',\n b' of',\n b' an',\n b' Enterprise',\n b' backup',\n b' system',\n b' including',\n b' systems',\n b' design',\n b' business',\n b'case',\n b' documentation',\n b' cost',\n b' benefit',\n b' analysis',\n b' technical',\n b' diagrams',\n b' and',\n b' work',\n b' flow',\n b' documentation',\n b' ',\n b' Review',\n b'ing',\n b' evaluating',\n b' and',\n b' analyzing',\n b' department',\n b'al',\n b' policies',\n b' guidelines',\n b' procedures',\n b' and',\n b' standards',\n b' with',\n b' management',\n b' and',\n b' staff',\n b' ',\n b' Developing',\n b' test',\n b' scripts',\n b' for',\n b' acceptance',\n b' unit',\n b' and',\n b' system',\n b' testing',\n b' of',\n b' Hyper',\n b'ion',\n b' Phase',\n b' ',\n b'1',\n b' and',\n b' Miami',\n b'Biz',\n b' Phase',\n b' ',\n b'2',\n b' ',\n b' Developing',\n b' Quality',\n b' Assurance',\n b' and',\n b' testing',\n b' plan',\n b' for',\n b' Hyper',\n b'ion',\n b' Phase',\n b' ',\n b'1',\n b' and',\n b' Miami',\n b'Biz',\n b' Phase',\n b' ',\n b'2',\n b' ',\n b' Debug',\n b'ging',\n b' and',\n b' logging',\n b' of',\n b' errors',\n b' in',\n b' Hyper',\n b'ion',\n b' and',\n b' Miami',\n b'Biz',\n b' using',\n b' Team',\n b' Foundation',\n b' Server',\n b' T',\n b'FS',\n b' ',\n b' Particip',\n b'ated',\n b' in',\n b' various',\n b' phases',\n b' of',\n b' the',\n b' project',\n b' life',\n b' cycle',\n b' such',\n b' as',\n b' determining',\n b' requirements',\n b' design',\n b' conceptual',\n b'ization',\n b' testing',\n b' implementation',\n b' deployment',\n b' and',\n b' release',\n b' for',\n b' the',\n b' Hyper',\n b'ion',\n b' and',\n b' Miami',\n b'Biz',\n b' projects',\n b' ',\n b' Collabor',\n b'ating',\n b' on',\n b' project',\n b' plans',\n b' for',\n b' Hyper',\n b'ion',\n b' and',\n b' Miami',\n b'Biz',\n b' ',\n b' Pre',\n b'paring',\n b' presentations',\n b' and',\n b' documentation',\n b' to',\n b' demonstrate',\n b' Hyper',\n b'ion',\n b' and',\n b' Miami',\n b'Biz',\n b' functionality',\n b' or',\n b' design',\n b' ',\n b' Monitoring',\n b' network',\n b' traffic',\n b' and',\n b' compiling',\n b' and',\n b' generating',\n b' statistical',\n b' information',\n b' using',\n b' Solar',\n b' Winds',\n b' ',\n b' Collabor',\n b'ating',\n b' on']" + "text/plain": [ + "[b'IN',\n", + " b'FORMATION',\n", + " b' TECHNO',\n", + " b'LOGY',\n", + " b' TECH',\n", + " b'NIC',\n", + " b'IAN',\n", + " b' I',\n", + " b' Summary',\n", + " b' Vers',\n", + " b'atile',\n", + " b' Systems',\n", + " b' Administrator',\n", + " b' possessing',\n", + " b' superior',\n", + " b' troubleshooting',\n", + " b' skills',\n", + " b' for',\n", + " b' networking',\n", + " b' issues',\n", + " b' end',\n", + " b' user',\n", + " b' problems',\n", + " b' and',\n", + " b' network',\n", + " b' security',\n", + " b' Experienced',\n", + " b' in',\n", + " b' server',\n", + " b' management',\n", + " b' systems',\n", + " b' analysis',\n", + " b' and',\n", + " b' offering',\n", + " b' inde',\n", + " b'pth',\n", + " b' understanding',\n", + " b' of',\n", + " b' IT',\n", + " b' infrastructure',\n", + " b' areas',\n", + " b' Detail',\n", + " b'oriented',\n", + " b' independent',\n", + " b' and',\n", + " b' focused',\n", + " b' on',\n", + " b' taking',\n", + " b' a',\n", + " b' systematic',\n", + " b' approach',\n", + " b' to',\n", + " b' solving',\n", + " b' complex',\n", + " b' problems',\n", + " b' Demonstr',\n", + " b'ated',\n", + " b' exceptional',\n", + " b' technical',\n", + " b' knowledge',\n", + " b' and',\n", + " b' skills',\n", + " b' while',\n", + " b' working',\n", + " b' with',\n", + " b' various',\n", + " b' teams',\n", + " b' to',\n", + " b' achieve',\n", + " b' shared',\n", + " b' goals',\n", + " b' and',\n", + " b' objectives',\n", + " b' Highlights',\n", + " b' ',\n", + " b' Active',\n", + " b' Directory',\n", + " b' ',\n", + " b' New',\n", + " b' technology',\n", + " b' and',\n", + " b' product',\n", + " b' research',\n", + " b' ',\n", + " b' Group',\n", + " b' Policy',\n", + " b' Objects',\n", + " b' ',\n", + " b' Office',\n", + " b' ',\n", + " b'365',\n", + " b' and',\n", + " b' Azure',\n", + " b' ',\n", + " b' PowerShell',\n", + " b' and',\n", + " b' VB',\n", + " b'Script',\n", + " b' ',\n", + " b' Storage',\n", + " b' management',\n", + " b' ',\n", + " b' Microsoft',\n", + " b' Exchange',\n", + " b' ',\n", + " b' Enterprise',\n", + " b' backup',\n", + " b' management',\n", + " b' ',\n", + " b' VM',\n", + " b'Ware',\n", + " b' experience',\n", + " b' ',\n", + " b' Disaster',\n", + " b' recovery',\n", + " b' Experience',\n", + " b' Information',\n", + " b' Technology',\n", + " b' Technician',\n", + " b' I',\n", + " b' Aug',\n", + " b' ',\n", + " b'200',\n", + " b'7',\n", + " b' to',\n", + " b' Current',\n", + " b' Company',\n", + " b' Name',\n", + " b' i',\n", + " b'14',\n", + " b' City',\n", + " b' State',\n", + " b' ',\n", + " b' M',\n", + " b'igr',\n", + " b'ating',\n", + " b' and',\n", + " b' managing',\n", + " b' user',\n", + " b' accounts',\n", + " b' in',\n", + " b' Microsoft',\n", + " b' Office',\n", + " b' ',\n", + " b'365',\n", + " b' and',\n", + " b' Exchange',\n", + " b' Online',\n", + " b' ',\n", + " b' Creating',\n", + " b' and',\n", + " b' managing',\n", + " b' virtual',\n", + " b' machines',\n", + " b' for',\n", + " b' systems',\n", + " b' such',\n", + " b' as',\n", + " b' domain',\n", + " b' controllers',\n", + " b' and',\n", + " b' Active',\n", + " b' Directory',\n", + " b' Federation',\n", + " b' Services',\n", + " b' A',\n", + " b'DFS',\n", + " b' in',\n", + " b' Microsoft',\n", + " b' Windows',\n", + " b' Azure',\n", + " b' I',\n", + " b'aaS',\n", + " b' ',\n", + " b' Creating',\n", + " b' and',\n", + " b' managing',\n", + " b' storage',\n", + " b' in',\n", + " b' Microsoft',\n", + " b' Windows',\n", + " b' Azure',\n", + " b' I',\n", + " b'aaS',\n", + " b' ',\n", + " b' Installing',\n", + " b' and',\n", + " b' configuring',\n", + " b' St',\n", + " b'or',\n", + " b'Simple',\n", + " b' i',\n", + " b'SC',\n", + " b'SI',\n", + " b' cloud',\n", + " b' array',\n", + " b' ST',\n", + " b'aa',\n", + " b'SB',\n", + " b'aaS',\n", + " b' ',\n", + " b' Installing',\n", + " b' configuring',\n", + " b' and',\n", + " b' testing',\n", + " b' Twin',\n", + " b'str',\n", + " b'ata',\n", + " b' i',\n", + " b'SC',\n", + " b'SI',\n", + " b' cloud',\n", + " b' array',\n", + " b' ST',\n", + " b'aa',\n", + " b'SB',\n", + " b'aaS',\n", + " b' ',\n", + " b' Collabor',\n", + " b'ating',\n", + " b' on',\n", + " b' project',\n", + " b' plan',\n", + " b' for',\n", + " b' Office',\n", + " b' ',\n", + " b'365',\n", + " b' migration',\n", + " b' ',\n", + " b' Developing',\n", + " b' detailed',\n", + " b' specifications',\n", + " b' for',\n", + " b' the',\n", + " b' Office',\n", + " b' ',\n", + " b'365',\n", + " b' migration',\n", + " b' including',\n", + " b' business',\n", + " b'case',\n", + " b' documentation',\n", + " b' cost',\n", + " b' benefit',\n", + " b' analyses',\n", + " b' technical',\n", + " b' diagrams',\n", + " b' and',\n", + " b' work',\n", + " b' flow',\n", + " b' documentation',\n", + " b' ',\n", + " b' Received',\n", + " b' training',\n", + " b' in',\n", + " b' MVC',\n", + " b' ',\n", + " b'4',\n", + " b' for',\n", + " b' Visual',\n", + " b' Studio',\n", + " b' using',\n", + " b' ',\n", + " b' Net',\n", + " b' Framework',\n", + " b' ',\n", + " b'445',\n", + " b' to',\n", + " b' develop',\n", + " b' application',\n", + " b' using',\n", + " b' HTML',\n", + " b'5',\n", + " b' and',\n", + " b' CSS',\n", + " b'3',\n", + " b' ',\n", + " b' Installing',\n", + " b' configuring',\n", + " b' and',\n", + " b' supporting',\n", + " b' Linux',\n", + " b' machines',\n", + " b' for',\n", + " b' the',\n", + " b' open',\n", + " b' WiFi',\n", + " b' network',\n", + " b' project',\n", + " b' ',\n", + " b' Comp',\n", + " b'iling',\n", + " b' and',\n", + " b' generating',\n", + " b' statistical',\n", + " b' information',\n", + " b' concerning',\n", + " b' wireless',\n", + " b' network',\n", + " b' traffic',\n", + " b' using',\n", + " b' C',\n", + " b'act',\n", + " b'i',\n", + " b' ',\n", + " b' Config',\n", + " b'uring',\n", + " b' wireless',\n", + " b' LAN',\n", + " b' router',\n", + " b' networking',\n", + " b' and',\n", + " b' security',\n", + " b' access',\n", + " b' ',\n", + " b' Installing',\n", + " b' and',\n", + " b' configuring',\n", + " b' wireless',\n", + " b' certificates',\n", + " b' ',\n", + " b' Developing',\n", + " b' detailed',\n", + " b' specifications',\n", + " b' for',\n", + " b' the',\n", + " b' acquisition',\n", + " b' of',\n", + " b' an',\n", + " b' Enterprise',\n", + " b' backup',\n", + " b' system',\n", + " b' including',\n", + " b' systems',\n", + " b' design',\n", + " b' business',\n", + " b'case',\n", + " b' documentation',\n", + " b' cost',\n", + " b' benefit',\n", + " b' analysis',\n", + " b' technical',\n", + " b' diagrams',\n", + " b' and',\n", + " b' work',\n", + " b' flow',\n", + " b' documentation',\n", + " b' ',\n", + " b' Review',\n", + " b'ing',\n", + " b' evaluating',\n", + " b' and',\n", + " b' analyzing',\n", + " b' department',\n", + " b'al',\n", + " b' policies',\n", + " b' guidelines',\n", + " b' procedures',\n", + " b' and',\n", + " b' standards',\n", + " b' with',\n", + " b' management',\n", + " b' and',\n", + " b' staff',\n", + " b' ',\n", + " b' Developing',\n", + " b' test',\n", + " b' scripts',\n", + " b' for',\n", + " b' acceptance',\n", + " b' unit',\n", + " b' and',\n", + " b' system',\n", + " b' testing',\n", + " b' of',\n", + " b' Hyper',\n", + " b'ion',\n", + " b' Phase',\n", + " b' ',\n", + " b'1',\n", + " b' and',\n", + " b' Miami',\n", + " b'Biz',\n", + " b' Phase',\n", + " b' ',\n", + " b'2',\n", + " b' ',\n", + " b' Developing',\n", + " b' Quality',\n", + " b' Assurance',\n", + " b' and',\n", + " b' testing',\n", + " b' plan',\n", + " b' for',\n", + " b' Hyper',\n", + " b'ion',\n", + " b' Phase',\n", + " b' ',\n", + " b'1',\n", + " b' and',\n", + " b' Miami',\n", + " b'Biz',\n", + " b' Phase',\n", + " b' ',\n", + " b'2',\n", + " b' ',\n", + " b' Debug',\n", + " b'ging',\n", + " b' and',\n", + " b' logging',\n", + " b' of',\n", + " b' errors',\n", + " b' in',\n", + " b' Hyper',\n", + " b'ion',\n", + " b' and',\n", + " b' Miami',\n", + " b'Biz',\n", + " b' using',\n", + " b' Team',\n", + " b' Foundation',\n", + " b' Server',\n", + " b' T',\n", + " b'FS',\n", + " b' ',\n", + " b' Particip',\n", + " b'ated',\n", + " b' in',\n", + " b' various',\n", + " b' phases',\n", + " b' of',\n", + " b' the',\n", + " b' project',\n", + " b' life',\n", + " b' cycle',\n", + " b' such',\n", + " b' as',\n", + " b' determining',\n", + " b' requirements',\n", + " b' design',\n", + " b' conceptual',\n", + " b'ization',\n", + " b' testing',\n", + " b' implementation',\n", + " b' deployment',\n", + " b' and',\n", + " b' release',\n", + " b' for',\n", + " b' the',\n", + " b' Hyper',\n", + " b'ion',\n", + " b' and',\n", + " b' Miami',\n", + " b'Biz',\n", + " b' projects',\n", + " b' ',\n", + " b' Collabor',\n", + " b'ating',\n", + " b' on',\n", + " b' project',\n", + " b' plans',\n", + " b' for',\n", + " b' Hyper',\n", + " b'ion',\n", + " b' and',\n", + " b' Miami',\n", + " b'Biz',\n", + " b' ',\n", + " b' Pre',\n", + " b'paring',\n", + " b' presentations',\n", + " b' and',\n", + " b' documentation',\n", + " b' to',\n", + " b' demonstrate',\n", + " b' Hyper',\n", + " b'ion',\n", + " b' and',\n", + " b' Miami',\n", + " b'Biz',\n", + " b' functionality',\n", + " b' or',\n", + " b' design',\n", + " b' ',\n", + " b' Monitoring',\n", + " b' network',\n", + " b' traffic',\n", + " b' and',\n", + " b' compiling',\n", + " b' and',\n", + " b' generating',\n", + " b' statistical',\n", + " b' information',\n", + " b' using',\n", + " b' Solar',\n", + " b' Winds',\n", + " b' ',\n", + " b' Collabor',\n", + " b'ating',\n", + " b' on']" + ] }, - "metadata": {}, "execution_count": 14, + "metadata": {}, "output_type": "execute_result" } ], - "execution_count": 14 + "source": [ + "import tiktoken\n", + "tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n", + "sample_encode = tokenizer.encode(df.chunk_text[0]) \n", + "decode = tokenizer.decode_tokens_bytes(sample_encode)\n", + "decode\n" + ] }, { "cell_type": "code", - "source": [ - "len(decode)" - ], + "execution_count": 15, "metadata": { "azdata_cell_guid": "31d80228-e140-413a-b1e3-c320a42b1f6c", "language": "python" @@ -442,17 +1062,25 @@ "outputs": [ { "data": { - "text/plain": "500" + "text/plain": [ + "500" + ] }, - "metadata": {}, "execution_count": 15, + "metadata": {}, "output_type": "execute_result" } ], - "execution_count": 15 + "source": [ + "len(decode)" + ] }, { + "attachments": {}, "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "ac19ee49-763a-4e2e-8c31-9ce1ba77bbe4" + }, "source": [ "# **PART 2 : Generating Embeddings for Text Chunks using Azure Open AI**\n", "\n", @@ -461,14 +1089,85 @@ "- We will use the Azure OpenAI API to generate these embeddings. The `get_embedding` function defined below takes a piece of text as input and returns its embedding using the `text-embedding-small` model\n", "\n", "- Ensure the Environment Variables are set correctly in the .env file" - ], - "metadata": { - "azdata_cell_guid": "ac19ee49-763a-4e2e-8c31-9ce1ba77bbe4" - }, - "attachments": {} + ] }, { "cell_type": "code", + "execution_count": 17, + "metadata": { + "azdata_cell_guid": "3aeda057-d0c0-40cd-bdcf-723abfed94e2", + "language": "python" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Completed 50 rows\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Completed 100 rows\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Completed 150 rows\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Completed 200 rows\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Completed 250 rows\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Completed 300 rows\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " filename chunkid \\\n", + "0 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 0_0 \n", + "1 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 0_1 \n", + "2 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 0_2 \n", + "3 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 1_0 \n", + "4 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 1_1 \n", + "\n", + " chunk \\\n", + "0 INFORMATION TECHNOLOGY TECHNICIAN I Summary Ve... \n", + "1 Disaster Recovery plan and procedures Researc... \n", + "2 Installing configuring and supporting McAfee a... \n", + "3 INFORMATION TECHNOLOGY MANAGER Professional Su... \n", + "4 network which entailed changing software and L... \n", + "\n", + " embedding \n", + "0 [-0.009804371, -0.0077886474, -0.0043056956, -... \n", + "1 [-0.0055441344, -0.0042689154, 0.00038358863, ... \n", + "2 [-0.005266213, -0.0009840217, -0.00897835, -0.... \n", + "3 [-0.0156019265, -0.007396069, 0.0030721354, -0... \n", + "4 [-0.0134326555, -0.011126321, 0.023804175, -0.... \n" + ] + } + ], "source": [ "import os\n", "import requests\n", @@ -537,75 +1236,44 @@ " 'embedding': all_embeddings\n", "})\n", "\n", - "print(result_df.head(5)) # Display the first few rows of the dataframe\n", - "" - ], - "metadata": { - "azdata_cell_guid": "3aeda057-d0c0-40cd-bdcf-723abfed94e2", - "language": "python" - }, - "outputs": [ - { - "name": "stdout", - "text": "Completed 50 rows\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Completed 100 rows\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Completed 150 rows\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Completed 200 rows\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Completed 250 rows\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": "Completed 300 rows\n", - "output_type": "stream" - }, - { - "name": "stdout", - "text": " filename chunkid \\\n0 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 0_0 \n1 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 0_1 \n2 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 0_2 \n3 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 1_0 \n4 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 1_1 \n\n chunk \\\n0 INFORMATION TECHNOLOGY TECHNICIAN I Summary Ve... \n1 Disaster Recovery plan and procedures Researc... \n2 Installing configuring and supporting McAfee a... \n3 INFORMATION TECHNOLOGY MANAGER Professional Su... \n4 network which entailed changing software and L... \n\n embedding \n0 [-0.009804371, -0.0077886474, -0.0043056956, -... \n1 [-0.0055441344, -0.0042689154, 0.00038358863, ... \n2 [-0.005266213, -0.0009840217, -0.00897835, -0.... \n3 [-0.0156019265, -0.007396069, 0.0030721354, -0... \n4 [-0.0134326555, -0.011126321, 0.023804175, -0.... \n", - "output_type": "stream" - } - ], - "execution_count": 17 + "print(result_df.head(5)) # Display the first few rows of the dataframe\n" + ] }, { + "attachments": {}, "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "04d42351-41ec-4ce9-9778-fe4ea2ecb8a2" + }, "source": [ "# **PART 3 : Using Azure SQL DB as a Vector Database to store and query embeddings**\n", "\n", "### **Load the embeddings into the Vector Database : Azure SQL DB**\n", "\n", "First let us define a function to connect to Azure SQLDB" - ], - "metadata": { - "azdata_cell_guid": "04d42351-41ec-4ce9-9778-fe4ea2ecb8a2" - }, - "attachments": {} + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "930a63bc-4c08-4205-b152-b1ad5c82057a", + "language": "python" + }, + "outputs": [], "source": [ - "#lets define a function to connect to SQLDB\n", + "# Connect to Azure SQL using mssql-python — the first-party Microsoft Python\n", + "# driver. Native Entra ID support (no manual token struct) and no ODBC driver\n", + "# install required. See https://github.com/microsoft/mssql-python\n", + "#\n", + "# pip install mssql-python\n", + "#\n", + "# If you prefer pyodbc, the old pyodbc + struct-token pattern still works —\n", + "# see the git history of this file for the previous version.\n", + "\n", "import os\n", "from dotenv import load_dotenv\n", - "import pyodbc\n", - "import struct\n", - "from azure.identity import DefaultAzureCredential\n", + "import mssql_python\n", "\n", "# Load environment variables from .env file\n", "load_dotenv()\n", @@ -613,54 +1281,53 @@ "def get_mssql_connection():\n", " # Retrieve the connection string from the environment variables\n", " entra_connection_string = os.getenv('ENTRA_CONNECTION_STRING')\n", - " sql_connection_string = os.getenv('SQL_CONNECTION_STRING')\n", + " sql_connection_string = os.getenv('SQL_CONNECTION_STRING')\n", "\n", - " # Determine the authentication method and connect to the database\n", " if entra_connection_string:\n", - " # Entra ID Service Principal Authentication\n", - " credential = DefaultAzureCredential(exclude_interactive_browser_credential=False) \n", - " token = credential.get_token('https://database.windows.net/.default')\n", - " token_bytes = token.token.encode('UTF-16LE')\n", - " token_struct = struct.pack(f' Date: Fri, 24 Jul 2026 11:45:34 -0400 Subject: [PATCH 2/5] Address Copilot PR review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - FineFoodReviews/000-setup.sql + 002-diskann-and-fulltext.sql: name the primary-key constraint explicitly (PK_reviews) so the fulltext KEY INDEX binding is deterministic. Avoids the auto-generated PK__reviews__ name that would otherwise fail on any fresh database. - FineFoodReviews/_load-reviews.py + _embed-reviews.py: replace hard-coded server/database/CSV path with MSSQL_SERVER / MSSQL_DATABASE / REVIEWS_CSV env vars (with safe placeholder + repo-relative defaults). No more developer-specific paths in the sample. - FineFoodReviews/_embed-reviews.py: clarify SET LOCK_TIMEOUT comment — it is a lock-wait cap, not a query/command timeout. - diskann-quickstart-azure-sql.sql + -sql-server-2025.sql: replace culture-sensitive FORMAT(..., 'N3') filler-vector formatting with CONVERT(varchar(5), CAST(x AS decimal(4,3))). Locale-invariant "." decimal separator, so the JSON-to-VECTOR cast is safe on any collation. - RAG-with-Documents/RAG-with-resumes.ipynb: clear all executed cell outputs and execution counts to shrink the diff and avoid checking in local run content (1754 -> 841 lines). --- DiskANN/FineFoodReviews/000-setup.sql | 2 +- .../002-diskann-and-fulltext.sql | 2 +- DiskANN/FineFoodReviews/_embed-reviews.py | 17 +- DiskANN/FineFoodReviews/_load-reviews.py | 23 +- DiskANN/diskann-quickstart-azure-sql.sql | 13 +- .../diskann-quickstart-sql-server-2025.sql | 13 +- RAG-with-Documents/RAG-with-resumes.ipynb | 2591 ++++++----------- 7 files changed, 883 insertions(+), 1778 deletions(-) diff --git a/DiskANN/FineFoodReviews/000-setup.sql b/DiskANN/FineFoodReviews/000-setup.sql index 22fd230..fd8d18b 100644 --- a/DiskANN/FineFoodReviews/000-setup.sql +++ b/DiskANN/FineFoodReviews/000-setup.sql @@ -17,7 +17,7 @@ drop table if exists dbo.reviews; go create table dbo.reviews ( - Id int not null primary key, + Id int not null constraint PK_reviews primary key, Time bigint null, ProductId nvarchar(50) null, UserId nvarchar(50) null, diff --git a/DiskANN/FineFoodReviews/002-diskann-and-fulltext.sql b/DiskANN/FineFoodReviews/002-diskann-and-fulltext.sql index 4f416d9..fa1ded0 100644 --- a/DiskANN/FineFoodReviews/002-diskann-and-fulltext.sql +++ b/DiskANN/FineFoodReviews/002-diskann-and-fulltext.sql @@ -59,7 +59,7 @@ create fulltext catalog ft_reviews_catalog as default; go create fulltext index on dbo.reviews (combined language 1033) - key index PK__reviews -- adjust if the primary-key index has a different auto-name + key index PK_reviews -- named explicitly in 000-setup.sql on ft_reviews_catalog with change_tracking auto; go diff --git a/DiskANN/FineFoodReviews/_embed-reviews.py b/DiskANN/FineFoodReviews/_embed-reviews.py index 3287fa1..eb28789 100644 --- a/DiskANN/FineFoodReviews/_embed-reviews.py +++ b/DiskANN/FineFoodReviews/_embed-reviews.py @@ -1,10 +1,15 @@ """Batch-embed dbo.reviews in chunks of 50 to avoid REST endpoint timeout. -Uses mssql-python (the native Microsoft first-party Python driver).""" -import sys, time +Uses mssql-python (the native Microsoft first-party Python driver). + +Configure via environment variables (or edit the defaults below): + MSSQL_SERVER e.g. myserver.database.windows.net + MSSQL_DATABASE e.g. FineFoodReviews +""" +import os, sys, time import mssql_python -SERVER = "antho-test-server.database.windows.net" -DATABASE = "VSLive2026" +SERVER = os.getenv("MSSQL_SERVER", ".database.windows.net") +DATABASE = os.getenv("MSSQL_DATABASE", "FineFoodReviews") def connect(): return mssql_python.connect( @@ -16,7 +21,9 @@ def connect(): conn = connect() cur = conn.cursor() -# Set a long command timeout since AI_GENERATE_EMBEDDINGS is external +# Lock timeout: cap how long DML on this session waits on row locks. This does +# not control external-REST timeout for AI_GENERATE_EMBEDDINGS; that is handled +# by retrying in the loop below. cur.execute("SET LOCK_TIMEOUT 60000;") start = time.time() diff --git a/DiskANN/FineFoodReviews/_load-reviews.py b/DiskANN/FineFoodReviews/_load-reviews.py index bd7791f..d67a545 100644 --- a/DiskANN/FineFoodReviews/_load-reviews.py +++ b/DiskANN/FineFoodReviews/_load-reviews.py @@ -1,13 +1,22 @@ -"""Load first 500 rows of Datasets/Reviews.csv into dbo.reviews on VSLive2026, -then trigger AI_GENERATE_EMBEDDINGS server-side. Uses mssql-python (the native -Microsoft first-party Python driver) with Entra Default auth — no token juggling.""" -import csv, sys, time +"""Load first 500 rows of Datasets/Reviews.csv into dbo.reviews, then +trigger AI_GENERATE_EMBEDDINGS server-side. Uses mssql-python (the native +Microsoft first-party Python driver) with Entra Default auth — no token juggling. + +Configure via environment variables (or edit the defaults below): + MSSQL_SERVER e.g. myserver.database.windows.net + MSSQL_DATABASE e.g. FineFoodReviews + REVIEWS_CSV path to Reviews.csv (defaults to ../../Datasets/Reviews.csv) +""" +import csv, os, sys, time from pathlib import Path import mssql_python -SERVER = "antho-test-server.database.windows.net" -DATABASE = "VSLive2026" -CSV = Path("/Users/annahoffman/azure-sql-db-vector-search/Datasets/Reviews.csv") +SERVER = os.getenv("MSSQL_SERVER", ".database.windows.net") +DATABASE = os.getenv("MSSQL_DATABASE", "FineFoodReviews") +CSV = Path(os.getenv( + "REVIEWS_CSV", + Path(__file__).resolve().parent.parent.parent / "Datasets" / "Reviews.csv", +)) N_ROWS = 500 def connect(): diff --git a/DiskANN/diskann-quickstart-azure-sql.sql b/DiskANN/diskann-quickstart-azure-sql.sql index 39538c7..fc70d2a 100644 --- a/DiskANN/diskann-quickstart-azure-sql.sql +++ b/DiskANN/diskann-quickstart-azure-sql.sql @@ -25,18 +25,19 @@ VALUES (10, 'AI Innovations', 'Latest innovations in AI.', '[0.4, 0.7, 0.2, 0.3, 0.1]'); GO --- Add 90 more rows with pseudo-random 5-dim vectors to satisfy the 100-row minimum for CREATE VECTOR INDEX +-- Add 90 more rows with pseudo-random 5-dim vectors to satisfy the 100-row minimum for CREATE VECTOR INDEX. +-- Numbers are formatted via CAST(... AS decimal(4,3)) which is locale-invariant (always '.'). INSERT INTO Articles (id, title, content, embedding) SELECT 10 + s.value AS id, CONCAT(N'Article ', 10 + s.value) AS title, CONCAT(N'Filler content ', 10 + s.value) AS content, CAST(CONCAT('[', - FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', - FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', - FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', - FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', - FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), + CONVERT(varchar(5), CAST(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0 AS decimal(4,3))), ',', + CONVERT(varchar(5), CAST(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0 AS decimal(4,3))), ',', + CONVERT(varchar(5), CAST(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0 AS decimal(4,3))), ',', + CONVERT(varchar(5), CAST(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0 AS decimal(4,3))), ',', + CONVERT(varchar(5), CAST(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0 AS decimal(4,3))), ']') AS VECTOR(5)) AS embedding FROM GENERATE_SERIES(1, 90) AS s; GO diff --git a/DiskANN/diskann-quickstart-sql-server-2025.sql b/DiskANN/diskann-quickstart-sql-server-2025.sql index ec90345..3bd13d4 100644 --- a/DiskANN/diskann-quickstart-sql-server-2025.sql +++ b/DiskANN/diskann-quickstart-sql-server-2025.sql @@ -38,18 +38,19 @@ VALUES (10, 'AI Innovations', 'Latest innovations in AI.', '[0.4, 0.7, 0.2, 0.3, 0.1]'); GO --- Add 90 more rows with pseudo-random 5-dim vectors to satisfy the 100-row minimum for CREATE VECTOR INDEX +-- Add 90 more rows with pseudo-random 5-dim vectors to satisfy the 100-row minimum for CREATE VECTOR INDEX. +-- Numbers are formatted via CAST(... AS decimal(4,3)) which is locale-invariant (always '.'). INSERT INTO Articles (id, title, content, embedding) SELECT 10 + s.value AS id, CONCAT(N'Article ', 10 + s.value) AS title, CONCAT(N'Filler content ', 10 + s.value) AS content, CAST(CONCAT('[', - FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', - FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', - FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', - FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), ',', - FORMAT(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0, 'N3'), + CONVERT(varchar(5), CAST(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0 AS decimal(4,3))), ',', + CONVERT(varchar(5), CAST(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0 AS decimal(4,3))), ',', + CONVERT(varchar(5), CAST(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0 AS decimal(4,3))), ',', + CONVERT(varchar(5), CAST(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0 AS decimal(4,3))), ',', + CONVERT(varchar(5), CAST(ABS(CHECKSUM(NEWID())) % 1000 / 1000.0 AS decimal(4,3))), ']') AS VECTOR(5)) AS embedding FROM GENERATE_SERIES(1, 90) AS s; GO diff --git a/RAG-with-Documents/RAG-with-resumes.ipynb b/RAG-with-Documents/RAG-with-resumes.ipynb index 7aafbc6..8d2ef70 100644 --- a/RAG-with-Documents/RAG-with-resumes.ipynb +++ b/RAG-with-Documents/RAG-with-resumes.ipynb @@ -1,1754 +1,841 @@ { - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "azdata_cell_guid": "507219d1-d713-4c41-86d5-e938bf69627c", - "language": "sql" - }, - "source": [ - "# Leveraging Azure SQL DB’s Native Vector Capabilities for Enhanced Resume Matching with Azure Document Intelligence and RAG\n", - "\n", - "In this tutorial, we will explore how to leverage Azure SQL DB’s new vector data type to store embeddings and perform similarity searches using built-in vector functions, enabling advanced resume matching to identify the most suitable candidates. \n", - "\n", - "By extracting and chunking content from PDF resumes using Azure Document Intelligence, generating embeddings with Azure OpenAI, and storing these embeddings in Azure SQL DB, we can perform sophisticated vector similarity searches and retrieval-augmented generation (RAG) to identify the most suitable candidates based on their resumes.\n", - "\n", - "### **Tutorial Overview**\n", - "\n", - "- This Python notebook will teach you to:\n", - " 1. **Chunk PDF Resumes**: Use **`Azure Document Intelligence`** to extract and chunk content from PDF resumes.\n", - " 2. **Create Embeddings**: Generate embeddings from the chunked content using the **`Azure OpenAI API`**.\n", - " 3. **Vector Database Utilization**: Store embeddings in **`Azure SQL DB`** utilizing the **`new Vector Data Type`** and perform similarity searches using built-in vector functions to find the most suitable candidates.\n", - " 4. **LLM Generation Augmentation**: Enhance language model generation with embeddings from a vector database. In this case, we use the embeddings to inform a GPT-4 chat model, enabling it to provide rich, context-aware answers about candidates based on their resumes\n", - "\n", - "## Dataset\n", - "\n", - "We use a sample dataset from [Kaggle](https://www.kaggle.com/datasets/snehaanbhawal/resume-dataset) containing PDF resumes for this tutorial. For the purpose of this tutorial we will use 120 resumes from the **Information-Technology** folder\n", - "\n", - "## Prerequisites\n", - "\n", - "- **Azure Subscription**: [Create one for free](https://azure.microsoft.com/free/cognitive-services?azure-portal=true)\n", - "- **Azure SQL Database**: [Set up your database for free](https://learn.microsoft.com/azure/azure-sql/database/free-offer?view=azuresql)\n", - "- **Azure Document Intelligence** [Create a FreeAzure Doc Intelligence resource](https:/learn.microsoft.com/azure/ai-services/document-intelligence/create-document-intelligence-resource?view=doc-intel-4.0.0)\n", - "- **Azure Data Studio**: Download [here](https://azure.microsoft.com/products/data-studio) to manage your Azure SQL database and [execute the notebook](https://learn.microsoft.com/azure-data-studio/notebooks/notebooks-python-kernel)\n", - "\n", - "## Additional Requirements for Embedding Generation\n", - "\n", - "- **Azure OpenAI Access**: Apply for access in the desired Azure subscription at [https://aka.ms/oai/access](https://aka.ms/oai/access)\n", - "- **Azure OpenAI Resource**: Deploy an embeddings model (e.g., `text-embedding-small` or `text-embedding-ada-002`) and a `GPT-4.0` model for chat completion. Refer to the [resource deployment guide](https://learn.microsoft.com/azure/ai-services/openai/how-to/create-resource)\n", - "- **Python**: Version 3.7.1 or later from Python.org. (Sample has been tested with Python 3.11)\n", - "- **Python Libraries**: Install the required libraries openai, num2words, matplotlib, plotly, scipy, scikit-learn, pandas, tiktoken, and mssql-python.\n", - "- **Jupyter Notebooks**: Use within [Azure Data Studio](https://learn.microsoft.com/en-us/azure-data-studio/notebooks/notebooks-guidance) or Visual Studio Code .\n", - "\n", - "Code snippets are adapted from the [Azure OpenAI Service embeddings Tutorial](https://learn.microsoft.com/en-us/azure/ai-services/openai/tutorials/embeddings?tabs=python-new%2Ccommand-line&pivots=programming-language-python)\n", - "\n", - "## Getting Started\n", - "\n", - "1. **Database Setup**: Execute SQL commands from the `createtable.sql` script to create the necessary table in your database.\n", - "2. **Model Deployment**: Deploy an embeddings model (`text-embedding-small` or `text-embedding-ada-002`) and a `GPT-4` model for chat completion. Note the 2 models deployment names for later use.\n", - "\n", - "![Deployed OpenAI Models](../Assets/modeldeployment.png)\n", - "\n", - "3. **Connection String**: Find your Azure SQL DB connection string in the Azure portal under your database settings.\n", - "4. **Configuration**: Populate the `.env` file with your SQL server connection details , Azure OpenAI key and endpoint, Azure Document Intelligence key and endpoint values.\n", - "\n", - "You can retrieve the Azure OpenAI _endpoint_ and _key_:\n", - "\n", - "![Azure OpenAI Endpoint and Key](../Assets/endpoint.png)\n", - "\n", - "You can [retrieve](https://learn.microsoft.com/azure/ai-services/document-intelligence/create-document-intelligence-resource?view=doc-intel-4.0.0#get-endpoint-url-and-keys) the Document Intelligence _endpoint_ and _key_:\n", - "\n", - "![Azure Document Intelligence Endpoint and Key](../Assets/docintelendpoint.png)\n", - "\n", - "## Running the Notebook\n", - "\n", - "To [execute the notebook](https://learn.microsoft.com/azure-data-studio/notebooks/notebooks-python-kernel), connect to your Azure SQL database using Azure Data Studio, which can be downloaded [here](https://azure.microsoft.com/products/data-studio)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "azdata_cell_guid": "fe37c601-5918-4055-badc-6c0ba90c68ce", - "language": "python" - }, - "outputs": [], - "source": [ - "#Setup the python libraries required for this notebook\n", - "#Please ensure that you navigate to the directory containing the `requirements.txt` file in your terminal\n", - "%pip install -r requirements.txt" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "azdata_cell_guid": "4c29709e-1c3a-495d-83ec-05737e220847", - "language": "python" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "#Load the env details\n", - "from dotenv import load_dotenv\n", - "load_dotenv()" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "azdata_cell_guid": "4b543f05-9036-4887-8737-09aa9f865ec2", - "language": "python" - }, - "source": [ - "# **PART 1: Extracting and Chunking Text from PDF Resumes using Azure Document Intelligence**\n", - "\n", - "Create an instance of the [DocumentAnalysisClient](https://learn.microsoft.com/azure/ai-services/document-intelligence/create-document-intelligence-resource?view=doc-intel-4.0.0#get-endpoint-url-and-keys) using the endpoint and API key. \n", - "\n", - "[Azure Document Intelligence](https://learn.microsoft.com/azure/ai-services/document-intelligence/?view=doc-intel-4.0.0_)(previously known as Form Recognizer) is a Azure cloud service that uses machine learning to analyze text and structured data from your documents. This client will be used to send requests to the [Azure Document Intelligence](https://learn.microsoft.com/python/api/overview/azure/ai-formrecognizer-readme?view=azure-python) service and receive responses containing the extracted text from the PDF resumes." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "azdata_cell_guid": "b20cc66a-50ce-4486-b275-d4683f4ba545", - "language": "python" - }, - "outputs": [], - "source": [ - "import os\n", - "import re\n", - "from azure.ai.formrecognizer import DocumentAnalysisClient\n", - "from azure.core.credentials import AzureKeyCredential\n", - "\n", - "# Load environment variables\n", - "endpoint = os.getenv(\"AZUREDOCINTELLIGENCE_ENDPOINT\")\n", - "api_key = os.getenv(\"AZUREDOCINTELLIGENCE_API_KEY\")\n", - "\n", - "# Create a DocumentAnalysisClient\n", - "document_analysis_client = DocumentAnalysisClient(\n", - " endpoint=endpoint,\n", - " credential=AzureKeyCredential(api_key)\n", - ")\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "azdata_cell_guid": "1ecc04b0-1a5f-4d48-819d-2cc06a070062", - "language": "python" - }, - "source": [ - "### **Analyze input documents using prebuilt model in Azure Document Intelligence**\n", - "\n", - "- DocumentAnalysisClient provides operations for analyzing input documents using prebuilt and custom models through the `begin_analyze_document` and `begin_analyze_document_from_url` APIs. In this tutorial we are using the [prebuilt-layout](https://learn.microsoft.com/python/api/overview/azure/ai-formrecognizer-readme?view=azure-python#using-prebuilt-models)\n", - " \n", - "\n", - "### **Split text into chunks of 500 tokens**\n", - "\n", - "- When faced with content that exceeds the embedding limit, we usually also chunk the content into smaller pieces and then embed those one at a time. Here we will use [tiktoken](https://github.com/openai/tiktoken?tab=readme-ov-file) to chunk the extracted text into token sizes of 500, as we will later pass the extracted chunks to to the `text-embedding-small` model for [generating text embeddings](https://learn.microsoft.com/azure/ai-services/openai/tutorials/embeddings?tabs=python-new%2Ccommand-line&pivots=programming-language-python) as this has a model input token limit of 8192.\n", - "\n", - "**Note**: You need to provide the location of the folder where the PDF files reside in the below script." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "azdata_cell_guid": "0355f92c-0546-4eac-aea8-b55d8bbef194", - "language": "python" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of PDF files in the directory: 120\n", - "Processing file 1/120: 92069209.pdf\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of chunks for file 92069209.pdf: 3\n", - "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 0_0, Chunk Length: 3035, Chunk Text: INFORMATION TECHNOLOGY TECHNICIAN I Summary Versat...\n", - "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 0_1, Chunk Length: 2959, Chunk Text: Disaster Recovery plan and procedures Researching...\n", - "File: 92069209.pdf, Chunk ID: 2, Unique Chunk ID: 0_2, Chunk Length: 2048, Chunk Text: Installing configuring and supporting McAfee antiv...\n", - "Processing file 2/120: 92069209.pdf\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of chunks for file 92069209.pdf: 3\n", - "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 1_0, Chunk Length: 3191, Chunk Text: INFORMATION TECHNOLOGY MANAGER Professional Summar...\n", - "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 1_1, Chunk Length: 2744, Chunk Text: network which entailed changing software and LAN c...\n", - "File: 92069209.pdf, Chunk ID: 2, Unique Chunk ID: 1_2, Chunk Length: 729, Chunk Text: i4 City State 2015 Master of Science Information ...\n", - "Processing file 3/120: 92069209.pdf\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of chunks for file 92069209.pdf: 2\n", - "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 2_0, Chunk Length: 3024, Chunk Text: WORKING RF SYSTEMS ENGINEER Qualifications Microso...\n", - "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 2_1, Chunk Length: 1892, Chunk Text: surveys ElectricalValidation Engineer May 2011 to ...\n", - "Processing file 4/120: 92069209.pdf\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of chunks for file 92069209.pdf: 2\n", - "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 3_0, Chunk Length: 2934, Chunk Text: INFORMATION TECHNOLOGY MANAGER Summary Dedicated I...\n", - "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 3_1, Chunk Length: 1917, Chunk Text: XP Vista and Mac operating systems Responsible fo...\n", - "Processing file 5/120: 92069209.pdf\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of chunks for file 92069209.pdf: 2\n", - "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 4_0, Chunk Length: 2854, Chunk Text: IT MANAGEMENT Career Overview Detailoriented profe...\n", - "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 4_1, Chunk Length: 2209, Chunk Text: to the device itself being recharged by a small so...\n", - "Processing file 6/120: 92069209.pdf\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of chunks for file 92069209.pdf: 2\n", - "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 5_0, Chunk Length: 2928, Chunk Text: INFORMATION TECHNOLOGY SPECIALIST Professional Sum...\n", - "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 5_1, Chunk Length: 1336, Chunk Text: business and or technical problems Applies metrics...\n", - "Processing file 7/120: 92069209.pdf\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of chunks for file 92069209.pdf: 3\n", - "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 6_0, Chunk Length: 3295, Chunk Text: BRANCH CHIEF INFORMATION TECHNOLOGY SPECIALIST Pro...\n", - "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 6_1, Chunk Length: 3179, Chunk Text: computer network operations plans including defens...\n", - "File: 92069209.pdf, Chunk ID: 2, Unique Chunk ID: 6_2, Chunk Length: 681, Chunk Text: 2009 DISA Action Officers Course 10Dec2009 DOD Inf...\n", - "Processing file 8/120: 92069209.pdf\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of chunks for file 92069209.pdf: 2\n", - "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 7_0, Chunk Length: 2784, Chunk Text: INFORMATION TECHNOLOGY COORDINATOR Career Overview...\n", - "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 7_1, Chunk Length: 2741, Chunk Text: dispatch process to an automated delivery system v...\n", - "Processing file 9/120: 92069209.pdf\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of chunks for file 92069209.pdf: 2\n", - "File: 92069209.pdf, Chunk ID: 0, Unique Chunk ID: 8_0, Chunk Length: 3013, Chunk Text: MANAGER INFORMATION TECHNOLOGY AND BUILDING AUTOM...\n", - "File: 92069209.pdf, Chunk ID: 1, Unique Chunk ID: 8_1, Chunk Length: 1972, Chunk Text: and Operation New Dawn focusing on network securit...\n", - "Processing file 10/120: 92069209.pdf\n" - ] - } - ], - "source": [ - "import os\n", - "import re\n", - "import pandas as pd\n", - "import tiktoken\n", - "\n", - "# Path to the directory containing PDF files\n", - "folder_path = os.path.join(os.getcwd(),'C:\\\\vectortest\\\\resumedata\\\\data\\\\data\\\\INFORMATION-TECHNOLOGY')\n", - "\n", - "def get_pdf_files(folder_path):\n", - " for path, subdirs, files in os.walk(folder_path):\n", - " for name in files:\n", - " if (name.endswith(\".pdf\")):\n", - " yield os.path.join(path, name)\n", - "\n", - "# Function to read PDF files and extract text using Azure AI Document Intelligence\n", - "def extract_text_from_pdf(pdf_path):\n", - " with open(pdf_path, \"rb\") as f:\n", - " poller = document_analysis_client.begin_analyze_document(\"prebuilt-layout\", document=f)\n", - " result = poller.result()\n", - " text = \"\"\n", - " for page in result.pages:\n", - " for line in page.lines:\n", - " text += line.content + \" \"\n", - " return text\n", - "\n", - "# Function to clean text and remove special characters\n", - "def clean_text(text):\n", - " text = re.sub(r'\\s+', ' ', text) # Remove extra whitespace\n", - " text = re.sub(r'[^a-zA-Z0-9\\s]', '', text) # Remove special characters\n", - " return text\n", - "\n", - "# Function to split text into chunks of 500 tokens\n", - "def split_text_into_token_chunks(text, max_tokens=500):\n", - " tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n", - " tokens = tokenizer.encode(text)\n", - " chunks = []\n", - " \n", - " for i in range(0, len(tokens), max_tokens):\n", - " chunk_tokens = tokens[i:i + max_tokens]\n", - " chunk_text = tokenizer.decode(chunk_tokens)\n", - " chunks.append(chunk_text)\n", - " \n", - " return chunks\n", - "\n", - "# Count the number of PDF files in the directory\n", - "pdf_files = [f for f in get_pdf_files(folder_path)]\n", - "num_files = len(pdf_files)\n", - "print(f\"Number of PDF files in the directory: {num_files}\")\n", - "\n", - "#Extract the file name\n", - "for pdf_file in pdf_files:\n", - " file_name = os.path.basename(pdf_file)\n", - "\n", - "# Create a DataFrame to store the chunks\n", - "data = []\n", - "\n", - "for file_id, pdf_file in enumerate(pdf_files):\n", - " print(f\"Processing file {file_id + 1}/{num_files}: {file_name}\")\n", - " pdf_path = os.path.join(folder_path, pdf_file)\n", - " text = extract_text_from_pdf(pdf_path)\n", - " cleaned_text = clean_text(text)\n", - " chunks = split_text_into_token_chunks(cleaned_text)\n", - " \n", - " print(f\"Number of chunks for file {file_name}: {len(chunks)}\")\n", - " \n", - " for chunk_id, chunk in enumerate(chunks):\n", - " chunk_text = chunk.strip() if chunk.strip() else \"NULL\"\n", - " unique_chunk_id = f\"{file_id}_{chunk_id}\"\n", - " print(f\"File: {file_name}, Chunk ID: {chunk_id}, Unique Chunk ID: {unique_chunk_id}, Chunk Length: {len(chunk_text)}, Chunk Text: {chunk_text[:50]}...\") # Print first 50 characters of chunk text\n", - " data.append({\n", - " \"file_name\": file_name,\n", - " \"chunk_id\": chunk_id,\n", - " \"chunk_text\": chunk_text,\n", - " \"unique_chunk_id\": unique_chunk_id\n", - " })\n", - "\n", - "df = pd.DataFrame(data)\n", - "df.head(3)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": { - "azdata_cell_guid": "3c0b7dba-c798-40d1-ac92-80288633497a", - "language": "python" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
file_namechunk_idchunk_textunique_chunk_idchunk_length
010089434.pdf0INFORMATION TECHNOLOGY TECHNICIAN I Summary Ve...0_03035
110089434.pdf1Disaster Recovery plan and procedures Researc...0_12959
210089434.pdf2Installing configuring and supporting McAfee a...0_22048
310247517.pdf0INFORMATION TECHNOLOGY MANAGER Professional Su...1_03191
410247517.pdf1network which entailed changing software and L...1_12744
\n", - "
" - ], - "text/plain": [ - " file_name chunk_id chunk_text \\\n", - "0 10089434.pdf 0 INFORMATION TECHNOLOGY TECHNICIAN I Summary Ve... \n", - "1 10089434.pdf 1 Disaster Recovery plan and procedures Researc... \n", - "2 10089434.pdf 2 Installing configuring and supporting McAfee a... \n", - "3 10247517.pdf 0 INFORMATION TECHNOLOGY MANAGER Professional Su... \n", - "4 10247517.pdf 1 network which entailed changing software and L... \n", - "\n", - " unique_chunk_id chunk_length \n", - "0 0_0 3035 \n", - "1 0_1 2959 \n", - "2 0_2 2048 \n", - "3 1_0 3191 \n", - "4 1_1 2744 " - ] - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "#read the top5 rows of the dataframe\n", - "df.head(5)\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "azdata_cell_guid": "655a8389-1018-4e80-a39d-85187cf3c46f", - "language": "python" - }, - "source": [ - "### **Tokenization vs. Character Length (OPTIONAL)**\n", - "\n", - "In this section, we will explore the difference between the character length of a text chunk and its tokenized representation. Character length simply counts the number of characters in a text, while tokenization breaks the text into meaningful units called tokens.\n", - "\n", - "Character Length First, let’s add a new column to our DataFrame to view the length of each chunk in terms of characters: Here, chunk\\_length represents the number of characters in each chunk." - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": { - "azdata_cell_guid": "5e171928-764a-4d61-899a-83a8e0c3ac79", - "language": "python" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " file_name chunk_id chunk_length\n", - "0 10089434.pdf 0 3035\n", - "1 10089434.pdf 1 2959\n", - "2 10089434.pdf 2 2048\n", - "3 10247517.pdf 0 3191\n", - "4 10247517.pdf 1 2744\n" - ] - } - ], - "source": [ - "# Add a new column 'chunk_length' to the DataFrame to view the length of each chunk\n", - "df['chunk_length'] = df['chunk_text'].apply(len)\n", - "\n", - "# Display the first few rows of the DataFrame with the new column\n", - "print(df[['file_name', 'chunk_id', 'chunk_length']].head(5))\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "azdata_cell_guid": "9b87f84b-149c-4eb5-b9f0-c1b55f6d606c", - "language": "python" - }, - "source": [ - "### Tokenization\n", - "To understand how text ultimately is tokenized, it can be helpful to run the below code: \n", - "\n", - "- We use the tiktoken library to tokenize the text. Tokenization breaks the text into smaller units, which can be words, subwords, or characters, depending on the tokenizer used. You can see that in some cases an entire word is represented with a single token whereas in others parts of words are split across multiple tokens. \n", - "\n", - "- If you then check the length of the decode variable, you'll find it matches 500 our specified token number. It is simply a way of making sure none of the data we pass to the model for tokenization and embedding exceeds the input token limit of 8,192\n", - "\n", - "- When we pass the documents to the embeddings model, it will break the documents into tokens similar (though not necessarily identical) to the examples below and then convert the tokens to a series of floating point numbers that will be accessible via vector search" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "azdata_cell_guid": "28188280-2bef-414a-a663-a017c944bc19", - "language": "python" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[b'IN',\n", - " b'FORMATION',\n", - " b' TECHNO',\n", - " b'LOGY',\n", - " b' TECH',\n", - " b'NIC',\n", - " b'IAN',\n", - " b' I',\n", - " b' Summary',\n", - " b' Vers',\n", - " b'atile',\n", - " b' Systems',\n", - " b' Administrator',\n", - " b' possessing',\n", - " b' superior',\n", - " b' troubleshooting',\n", - " b' skills',\n", - " b' for',\n", - " b' networking',\n", - " b' issues',\n", - " b' end',\n", - " b' user',\n", - " b' problems',\n", - " b' and',\n", - " b' network',\n", - " b' security',\n", - " b' Experienced',\n", - " b' in',\n", - " b' server',\n", - " b' management',\n", - " b' systems',\n", - " b' analysis',\n", - " b' and',\n", - " b' offering',\n", - " b' inde',\n", - " b'pth',\n", - " b' understanding',\n", - " b' of',\n", - " b' IT',\n", - " b' infrastructure',\n", - " b' areas',\n", - " b' Detail',\n", - " b'oriented',\n", - " b' independent',\n", - " b' and',\n", - " b' focused',\n", - " b' on',\n", - " b' taking',\n", - " b' a',\n", - " b' systematic',\n", - " b' approach',\n", - " b' to',\n", - " b' solving',\n", - " b' complex',\n", - " b' problems',\n", - " b' Demonstr',\n", - " b'ated',\n", - " b' exceptional',\n", - " b' technical',\n", - " b' knowledge',\n", - " b' and',\n", - " b' skills',\n", - " b' while',\n", - " b' working',\n", - " b' with',\n", - " b' various',\n", - " b' teams',\n", - " b' to',\n", - " b' achieve',\n", - " b' shared',\n", - " b' goals',\n", - " b' and',\n", - " b' objectives',\n", - " b' Highlights',\n", - " b' ',\n", - " b' Active',\n", - " b' Directory',\n", - " b' ',\n", - " b' New',\n", - " b' technology',\n", - " b' and',\n", - " b' product',\n", - " b' research',\n", - " b' ',\n", - " b' Group',\n", - " b' Policy',\n", - " b' Objects',\n", - " b' ',\n", - " b' Office',\n", - " b' ',\n", - " b'365',\n", - " b' and',\n", - " b' Azure',\n", - " b' ',\n", - " b' PowerShell',\n", - " b' and',\n", - " b' VB',\n", - " b'Script',\n", - " b' ',\n", - " b' Storage',\n", - " b' management',\n", - " b' ',\n", - " b' Microsoft',\n", - " b' Exchange',\n", - " b' ',\n", - " b' Enterprise',\n", - " b' backup',\n", - " b' management',\n", - " b' ',\n", - " b' VM',\n", - " b'Ware',\n", - " b' experience',\n", - " b' ',\n", - " b' Disaster',\n", - " b' recovery',\n", - " b' Experience',\n", - " b' Information',\n", - " b' Technology',\n", - " b' Technician',\n", - " b' I',\n", - " b' Aug',\n", - " b' ',\n", - " b'200',\n", - " b'7',\n", - " b' to',\n", - " b' Current',\n", - " b' Company',\n", - " b' Name',\n", - " b' i',\n", - " b'14',\n", - " b' City',\n", - " b' State',\n", - " b' ',\n", - " b' M',\n", - " b'igr',\n", - " b'ating',\n", - " b' and',\n", - " b' managing',\n", - " b' user',\n", - " b' accounts',\n", - " b' in',\n", - " b' Microsoft',\n", - " b' Office',\n", - " b' ',\n", - " b'365',\n", - " b' and',\n", - " b' Exchange',\n", - " b' Online',\n", - " b' ',\n", - " b' Creating',\n", - " b' and',\n", - " b' managing',\n", - " b' virtual',\n", - " b' machines',\n", - " b' for',\n", - " b' systems',\n", - " b' such',\n", - " b' as',\n", - " b' domain',\n", - " b' controllers',\n", - " b' and',\n", - " b' Active',\n", - " b' Directory',\n", - " b' Federation',\n", - " b' Services',\n", - " b' A',\n", - " b'DFS',\n", - " b' in',\n", - " b' Microsoft',\n", - " b' Windows',\n", - " b' Azure',\n", - " b' I',\n", - " b'aaS',\n", - " b' ',\n", - " b' Creating',\n", - " b' and',\n", - " b' managing',\n", - " b' storage',\n", - " b' in',\n", - " b' Microsoft',\n", - " b' Windows',\n", - " b' Azure',\n", - " b' I',\n", - " b'aaS',\n", - " b' ',\n", - " b' Installing',\n", - " b' and',\n", - " b' configuring',\n", - " b' St',\n", - " b'or',\n", - " b'Simple',\n", - " b' i',\n", - " b'SC',\n", - " b'SI',\n", - " b' cloud',\n", - " b' array',\n", - " b' ST',\n", - " b'aa',\n", - " b'SB',\n", - " b'aaS',\n", - " b' ',\n", - " b' Installing',\n", - " b' configuring',\n", - " b' and',\n", - " b' testing',\n", - " b' Twin',\n", - " b'str',\n", - " b'ata',\n", - " b' i',\n", - " b'SC',\n", - " b'SI',\n", - " b' cloud',\n", - " b' array',\n", - " b' ST',\n", - " b'aa',\n", - " b'SB',\n", - " b'aaS',\n", - " b' ',\n", - " b' Collabor',\n", - " b'ating',\n", - " b' on',\n", - " b' project',\n", - " b' plan',\n", - " b' for',\n", - " b' Office',\n", - " b' ',\n", - " b'365',\n", - " b' migration',\n", - " b' ',\n", - " b' Developing',\n", - " b' detailed',\n", - " b' specifications',\n", - " b' for',\n", - " b' the',\n", - " b' Office',\n", - " b' ',\n", - " b'365',\n", - " b' migration',\n", - " b' including',\n", - " b' business',\n", - " b'case',\n", - " b' documentation',\n", - " b' cost',\n", - " b' benefit',\n", - " b' analyses',\n", - " b' technical',\n", - " b' diagrams',\n", - " b' and',\n", - " b' work',\n", - " b' flow',\n", - " b' documentation',\n", - " b' ',\n", - " b' Received',\n", - " b' training',\n", - " b' in',\n", - " b' MVC',\n", - " b' ',\n", - " b'4',\n", - " b' for',\n", - " b' Visual',\n", - " b' Studio',\n", - " b' using',\n", - " b' ',\n", - " b' Net',\n", - " b' Framework',\n", - " b' ',\n", - " b'445',\n", - " b' to',\n", - " b' develop',\n", - " b' application',\n", - " b' using',\n", - " b' HTML',\n", - " b'5',\n", - " b' and',\n", - " b' CSS',\n", - " b'3',\n", - " b' ',\n", - " b' Installing',\n", - " b' configuring',\n", - " b' and',\n", - " b' supporting',\n", - " b' Linux',\n", - " b' machines',\n", - " b' for',\n", - " b' the',\n", - " b' open',\n", - " b' WiFi',\n", - " b' network',\n", - " b' project',\n", - " b' ',\n", - " b' Comp',\n", - " b'iling',\n", - " b' and',\n", - " b' generating',\n", - " b' statistical',\n", - " b' information',\n", - " b' concerning',\n", - " b' wireless',\n", - " b' network',\n", - " b' traffic',\n", - " b' using',\n", - " b' C',\n", - " b'act',\n", - " b'i',\n", - " b' ',\n", - " b' Config',\n", - " b'uring',\n", - " b' wireless',\n", - " b' LAN',\n", - " b' router',\n", - " b' networking',\n", - " b' and',\n", - " b' security',\n", - " b' access',\n", - " b' ',\n", - " b' Installing',\n", - " b' and',\n", - " b' configuring',\n", - " b' wireless',\n", - " b' certificates',\n", - " b' ',\n", - " b' Developing',\n", - " b' detailed',\n", - " b' specifications',\n", - " b' for',\n", - " b' the',\n", - " b' acquisition',\n", - " b' of',\n", - " b' an',\n", - " b' Enterprise',\n", - " b' backup',\n", - " b' system',\n", - " b' including',\n", - " b' systems',\n", - " b' design',\n", - " b' business',\n", - " b'case',\n", - " b' documentation',\n", - " b' cost',\n", - " b' benefit',\n", - " b' analysis',\n", - " b' technical',\n", - " b' diagrams',\n", - " b' and',\n", - " b' work',\n", - " b' flow',\n", - " b' documentation',\n", - " b' ',\n", - " b' Review',\n", - " b'ing',\n", - " b' evaluating',\n", - " b' and',\n", - " b' analyzing',\n", - " b' department',\n", - " b'al',\n", - " b' policies',\n", - " b' guidelines',\n", - " b' procedures',\n", - " b' and',\n", - " b' standards',\n", - " b' with',\n", - " b' management',\n", - " b' and',\n", - " b' staff',\n", - " b' ',\n", - " b' Developing',\n", - " b' test',\n", - " b' scripts',\n", - " b' for',\n", - " b' acceptance',\n", - " b' unit',\n", - " b' and',\n", - " b' system',\n", - " b' testing',\n", - " b' of',\n", - " b' Hyper',\n", - " b'ion',\n", - " b' Phase',\n", - " b' ',\n", - " b'1',\n", - " b' and',\n", - " b' Miami',\n", - " b'Biz',\n", - " b' Phase',\n", - " b' ',\n", - " b'2',\n", - " b' ',\n", - " b' Developing',\n", - " b' Quality',\n", - " b' Assurance',\n", - " b' and',\n", - " b' testing',\n", - " b' plan',\n", - " b' for',\n", - " b' Hyper',\n", - " b'ion',\n", - " b' Phase',\n", - " b' ',\n", - " b'1',\n", - " b' and',\n", - " b' Miami',\n", - " b'Biz',\n", - " b' Phase',\n", - " b' ',\n", - " b'2',\n", - " b' ',\n", - " b' Debug',\n", - " b'ging',\n", - " b' and',\n", - " b' logging',\n", - " b' of',\n", - " b' errors',\n", - " b' in',\n", - " b' Hyper',\n", - " b'ion',\n", - " b' and',\n", - " b' Miami',\n", - " b'Biz',\n", - " b' using',\n", - " b' Team',\n", - " b' Foundation',\n", - " b' Server',\n", - " b' T',\n", - " b'FS',\n", - " b' ',\n", - " b' Particip',\n", - " b'ated',\n", - " b' in',\n", - " b' various',\n", - " b' phases',\n", - " b' of',\n", - " b' the',\n", - " b' project',\n", - " b' life',\n", - " b' cycle',\n", - " b' such',\n", - " b' as',\n", - " b' determining',\n", - " b' requirements',\n", - " b' design',\n", - " b' conceptual',\n", - " b'ization',\n", - " b' testing',\n", - " b' implementation',\n", - " b' deployment',\n", - " b' and',\n", - " b' release',\n", - " b' for',\n", - " b' the',\n", - " b' Hyper',\n", - " b'ion',\n", - " b' and',\n", - " b' Miami',\n", - " b'Biz',\n", - " b' projects',\n", - " b' ',\n", - " b' Collabor',\n", - " b'ating',\n", - " b' on',\n", - " b' project',\n", - " b' plans',\n", - " b' for',\n", - " b' Hyper',\n", - " b'ion',\n", - " b' and',\n", - " b' Miami',\n", - " b'Biz',\n", - " b' ',\n", - " b' Pre',\n", - " b'paring',\n", - " b' presentations',\n", - " b' and',\n", - " b' documentation',\n", - " b' to',\n", - " b' demonstrate',\n", - " b' Hyper',\n", - " b'ion',\n", - " b' and',\n", - " b' Miami',\n", - " b'Biz',\n", - " b' functionality',\n", - " b' or',\n", - " b' design',\n", - " b' ',\n", - " b' Monitoring',\n", - " b' network',\n", - " b' traffic',\n", - " b' and',\n", - " b' compiling',\n", - " b' and',\n", - " b' generating',\n", - " b' statistical',\n", - " b' information',\n", - " b' using',\n", - " b' Solar',\n", - " b' Winds',\n", - " b' ',\n", - " b' Collabor',\n", - " b'ating',\n", - " b' on']" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import tiktoken\n", - "tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n", - "sample_encode = tokenizer.encode(df.chunk_text[0]) \n", - "decode = tokenizer.decode_tokens_bytes(sample_encode)\n", - "decode\n" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "azdata_cell_guid": "31d80228-e140-413a-b1e3-c320a42b1f6c", - "language": "python" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "500" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(decode)" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "azdata_cell_guid": "ac19ee49-763a-4e2e-8c31-9ce1ba77bbe4" - }, - "source": [ - "# **PART 2 : Generating Embeddings for Text Chunks using Azure Open AI**\n", - "\n", - "- After extracting and chunking the text from PDF resumes, we will generate embeddings for each chunk. These embeddings are numerical representations of the text that capture its semantic meaning. By creating embeddings for the text chunks, we can perform advanced similarity searches and enhance language model generation.\n", - "\n", - "- We will use the Azure OpenAI API to generate these embeddings. The `get_embedding` function defined below takes a piece of text as input and returns its embedding using the `text-embedding-small` model\n", - "\n", - "- Ensure the Environment Variables are set correctly in the .env file" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "azdata_cell_guid": "3aeda057-d0c0-40cd-bdcf-723abfed94e2", - "language": "python" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Completed 50 rows\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Completed 100 rows\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Completed 150 rows\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Completed 200 rows\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Completed 250 rows\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Completed 300 rows\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - " filename chunkid \\\n", - "0 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 0_0 \n", - "1 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 0_1 \n", - "2 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 0_2 \n", - "3 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 1_0 \n", - "4 C:\\vectortest\\resumedata\\data\\data\\INFORMATION... 1_1 \n", - "\n", - " chunk \\\n", - "0 INFORMATION TECHNOLOGY TECHNICIAN I Summary Ve... \n", - "1 Disaster Recovery plan and procedures Researc... \n", - "2 Installing configuring and supporting McAfee a... \n", - "3 INFORMATION TECHNOLOGY MANAGER Professional Su... \n", - "4 network which entailed changing software and L... \n", - "\n", - " embedding \n", - "0 [-0.009804371, -0.0077886474, -0.0043056956, -... \n", - "1 [-0.0055441344, -0.0042689154, 0.00038358863, ... \n", - "2 [-0.005266213, -0.0009840217, -0.00897835, -0.... \n", - "3 [-0.0156019265, -0.007396069, 0.0030721354, -0... \n", - "4 [-0.0134326555, -0.011126321, 0.023804175, -0.... \n" - ] - } - ], - "source": [ - "import os\n", - "import requests\n", - "from num2words import num2words\n", - "import pandas as pd\n", - "import numpy as np\n", - "import json\n", - "from openai import AzureOpenAI\n", - "\n", - "# Specify your model name\n", - "openai_embedding_model = os.getenv(\"AZOPENAI_EMBEDDING_MODEL_DEPLOYMENT_NAME\")\n", - "\n", - "# Assuming openai_url and openai_key are your environment variables\n", - "openai_url = os.getenv(\"AZOPENAI_ENDPOINT\") + \"openai/deployments/\" + openai_embedding_model + \"/embeddings?api-version=2023-05-15\"\n", - "openai_key = os.getenv(\"AZOPENAI_API_KEY\")\n", - "\n", - "def get_embedding(text):\n", - " \"\"\"\n", - " Get sentence embedding using the Azure OpenAI text-embedding-small model.\n", - "\n", - " Args:\n", - " text (str): Text to embed.\n", - "\n", - " Returns:\n", - " list: A list containing the embedding.\n", - " \"\"\"\n", - " response = requests.post(openai_url,\n", - " headers={\"api-key\": openai_key, \"Content-Type\": \"application/json\"},\n", - " json={\"input\": [text]} # Embed the extracted chunk\n", - " )\n", - " \n", - " if response.status_code == 200:\n", - " response_json = response.json()\n", - " embedding = json.loads(str(response_json['data'][0]['embedding']))\n", - " return embedding\n", - " else:\n", - " return None\n", - "\n", - "# Example usage\n", - "all_filenames = []\n", - "all_chunkids = []\n", - "all_chunks = []\n", - "all_embeddings = []\n", - "\n", - "# Assuming df is already defined with the required columns\n", - "for index, row in df.iterrows():\n", - " filename = row['file_name']\n", - " chunkid = row['unique_chunk_id']\n", - " chunk = row['chunk_text']\n", - " embedding = get_embedding(chunk)\n", - " \n", - " if embedding is not None:\n", - " all_filenames.append(filename)\n", - " all_chunkids.append(chunkid)\n", - " all_chunks.append(chunk)\n", - " all_embeddings.append(embedding)\n", - " \n", - " if (index + 1) % 50 == 0: # Print progress every 50 rows\n", - " print(f\"Completed {index + 1} rows\")\n", - "\n", - "# Create a new DataFrame with the results\n", - "result_df = pd.DataFrame({\n", - " 'filename': all_filenames,\n", - " 'chunkid': all_chunkids,\n", - " 'chunk': all_chunks,\n", - " 'embedding': all_embeddings\n", - "})\n", - "\n", - "print(result_df.head(5)) # Display the first few rows of the dataframe\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "azdata_cell_guid": "04d42351-41ec-4ce9-9778-fe4ea2ecb8a2" - }, - "source": [ - "# **PART 3 : Using Azure SQL DB as a Vector Database to store and query embeddings**\n", - "\n", - "### **Load the embeddings into the Vector Database : Azure SQL DB**\n", - "\n", - "First let us define a function to connect to Azure SQLDB" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "azdata_cell_guid": "930a63bc-4c08-4205-b152-b1ad5c82057a", - "language": "python" - }, - "outputs": [], - "source": [ - "# Connect to Azure SQL using mssql-python — the first-party Microsoft Python\n", - "# driver. Native Entra ID support (no manual token struct) and no ODBC driver\n", - "# install required. See https://github.com/microsoft/mssql-python\n", - "#\n", - "# pip install mssql-python\n", - "#\n", - "# If you prefer pyodbc, the old pyodbc + struct-token pattern still works —\n", - "# see the git history of this file for the previous version.\n", - "\n", - "import os\n", - "from dotenv import load_dotenv\n", - "import mssql_python\n", - "\n", - "# Load environment variables from .env file\n", - "load_dotenv()\n", - "\n", - "def get_mssql_connection():\n", - " # Retrieve the connection string from the environment variables\n", - " entra_connection_string = os.getenv('ENTRA_CONNECTION_STRING')\n", - " sql_connection_string = os.getenv('SQL_CONNECTION_STRING')\n", - "\n", - " if entra_connection_string:\n", - " # Entra ID (passwordless) — mssql-python handles the token exchange for\n", - " # you via `authentication='ActiveDirectoryDefault'`, which chains\n", - " # DefaultAzureCredential (Azure CLI, VS Code, managed identity, etc.).\n", - " return mssql_python.connect(entra_connection_string,\n", - " authentication='ActiveDirectoryDefault')\n", - " elif sql_connection_string:\n", - " # SQL Authentication — connection string carries user + password\n", - " return mssql_python.connect(sql_connection_string)\n", - " else:\n", - " raise ValueError(\"No valid connection string found in the environment variables.\")\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "azdata_cell_guid": "e929c112-65d4-46f1-a9a2-7c9434b2d7cb", - "language": "python" - }, - "source": [ - "### **Insert embeddings into the native 'Vector' Data Type**\n", - "\n", - "We will insert our vectors into the SQL Table now. Azure SQL DB now has a dedicated, native, data type for storing vectors: the `vector` data type. Read about the preview [here](https://devblogs.microsoft.com/azure-sql/eap-for-vector-support-refresh-introducing-vector-type)\n", - "\n", - "The table embeddings has a column called vector which is vector(1536) type. Ensure you have created the table using the script `CreateTable.sql` before running the below code." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "azdata_cell_guid": "680259d9-77ce-4b63-b412-9bea33bb0f43", - "language": "python" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Data inserted successfully into the 'resumedocs' table.\n" - ] - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "# Retrieve the connection string from the function get_mssql_connection()\n", - "conn = get_mssql_connection()\n", - "\n", - "# Create a cursor object\n", - "cursor = conn.cursor()\n", - "\n", - "# Loop through the DataFrame rows and insert them into the table.\n", - "#\n", - "# The embedding is sent as a JSON string over the wire, then cast twice:\n", - "# NVARCHAR(MAX) first (mssql-python binds Python strings as ntext, which\n", - "# CAST cannot convert directly to VECTOR), then VECTOR(1536).\n", - "for index, row in result_df.iterrows():\n", - " chunkid = row['chunkid']\n", - " filename = row['filename']\n", - " chunk = row['chunk']\n", - " embedding = row['embedding']\n", - "\n", - " query = \"\"\"\n", - " INSERT INTO resumedocs (chunkid, filename, chunk, embedding)\n", - " VALUES (?, ?, ?, CAST(CAST(? AS NVARCHAR(MAX)) AS VECTOR(1536)))\n", - " \"\"\"\n", - " cursor.execute(query, chunkid, filename, chunk, json.dumps(embedding))\n", - "\n", - "# Commit the changes\n", - "conn.commit()\n", - "\n", - "print(\"Data inserted successfully into the 'resumedocs' table.\")\n", - "\n", - "# Close the connection\n", - "conn.close()\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "azdata_cell_guid": "ede638b9-d681-4cb9-9ab8-9651e3099a36", - "language": "python" - }, - "source": [ - "Let's take a look at the data in the Resume Docs table:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "azdata_cell_guid": "88d1a90e-e93c-426e-934d-fb1a47dfd900", - "language": "python" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+--------------+---------+----------------------+----------------------+\n", - "| filename | chunkid | chunk | embedding |\n", - "+--------------+---------+----------------------+----------------------+\n", - "| 10089434.pdf | 0_0 | INFORMATION TECHNOLO | [-9.8043708e-003,-7. |\n", - "| 10089434.pdf | 0_1 | Disaster Recovery pl | [-5.5441344e-003,-4. |\n", - "| 10089434.pdf | 0_2 | Installing configuri | [-5.2662129e-003,-9. |\n", - "| 10247517.pdf | 1_0 | INFORMATION TECHNOLO | [-1.5601926e-002,-7. |\n", - "| 10247517.pdf | 1_1 | network which entail | [-1.3432655e-002,-1. |\n", - "| 10247517.pdf | 1_2 | i4 City State 2015 M | [7.1866140e-003,-7.4 |\n", - "| 10265057.pdf | 2_0 | WORKING RF SYSTEMS E | [-1.8229846e-002,-6. |\n", - "| 10265057.pdf | 2_1 | surveys ElectricalVa | [-2.1091947e-002,2.1 |\n", - "| 10553553.pdf | 3_0 | INFORMATION TECHNOLO | [-9.1670882e-003,-1. |\n", - "| 10553553.pdf | 3_1 | XP Vista and Mac ope | [3.9297581e-004,-3.9 |\n", - "+--------------+---------+----------------------+----------------------+\n" - ] - } - ], - "source": [ - "from prettytable import PrettyTable\n", - "import pandas as pd\n", - "\n", - "# Load environment variables from .env file\n", - "load_dotenv()\n", - "\n", - "# Retrieve the connection using the mssql-python-backed helper\n", - "conn = get_mssql_connection()\n", - "\n", - "# Create a cursor object\n", - "cursor = conn.cursor()\n", - "\n", - "# Use placeholders for the parameters in the SQL query\n", - "query = \"SELECT TOP(10) filename, chunkid, chunk, CAST(embedding AS NVARCHAR(MAX)) as embedding FROM dbo.resumedocs ORDER BY Id\"\n", - "\n", - "# Execute the query with the parameters\n", - "cursor.execute(query)\n", - "queryresults = cursor.fetchall()\n", - "\n", - "# Get column names from cursor.description\n", - "column_names = [column[0] for column in cursor.description]\n", - "\n", - "# Create a PrettyTable object\n", - "table = PrettyTable()\n", - "\n", - "# Add column names to the table\n", - "table.field_names = column_names\n", - "\n", - "# Set max width for each column to truncate data\n", - "table.max_width = 20\n", - "\n", - "# Add rows to the table\n", - "for row in queryresults:\n", - " # Truncate each value to 20 characters\n", - " truncated_row = [str(value)[:20] for value in row]\n", - " table.add_row(truncated_row)\n", - "\n", - "# Print the table\n", - "print(table)\n", - "\n", - "# Commit the changes\n", - "conn.commit()\n", - "# Close the connection\n", - "conn.close()\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "azdata_cell_guid": "b7b0fda6-3322-4bbc-8d08-d0b567da79ec", - "language": "python" - }, - "source": [ - "### **Performing Vector Similarity Search in Azure SQL DB using VECTOR\\_DISTANCE built in function**\n", - "\n", - "Let's now query our ResumeDocs table to get the top similar candidates given the User search query.\n", - "\n", - "What we are doing: Given any user search query, we can obtain the vector representation of that text. We then use this vector to calculate the cosine distance against all the resume embeddings stored in the database. By selecting only the closest matches, we can identify the resumes most relevant to the user’s query. This helps in finding the most suitable candidates based on their resumes.\n", - "\n", - "The most common distance is the cosine similarity, which can be calculated quite easily in SQL with the help of the new distance functions.\n", - "\n", - "```\n", - "VECTOR_DISTANCE('distance metric', V1, V2)\n", - "\n", - "```\n", - "\n", - "We can use **cosine**, **euclidean**, and **dot** as the distance metric today.\n", - "\n", - "We will define the function `vector_search_sql`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "azdata_cell_guid": "1b4f0ca2-2401-4f90-a44d-75dce03500cc", - "language": "python" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[('20001721.pdf', '41_0', 'INFORMATION TECHNOLOGY STUDENT Career Overview Resultsdriven Database Administrator with extensive education in programming relational database management and computer technology maintenance Qualifications Database servers Structured query language SQL expert Programming and design skills Document management Strong collaborative skills Strong analytical skills Customer needs assessment Excellent problem solving skills Technical Skills Skills Experience Total Years Last Used Windows Unix Linux Mac OSX VMWare HTTPApache DNSBIND SSH SNMP DNS DHCP FTP Intermediate 2 May 2016 Accomplishments Customer Service Handled customers effectively by identifying needs quickly gaining trust approaching complex situations and resolving problems to maximize efficiency Data Preparation Prepared chain of custody packets for title sale reviews of procedures and feesservices justification Administration Performed administration tasks such as filing developing spreadsheets faxing reports photocopying collateral and scanning documents for interdepartmental use Reporting Maintained status reports to provide management with updated information for client projects Application Design Used objectoriented designprogramming to design new standalone application Planned installed configured and monitored document management infrastructure Coordinated scheduled software and hardware patches upgrades and enhancements to platforms Collaborated with IT teams to design and implement continuous process improvements to prevent production application incidents Work Experience Company Name January 2014 to Current INFORMATION TECHNOLOGY STUDENT City State Presented various projects including VPN RDMS and IT Proposals to several classes and instructors Worked independently and as part of a team to achieve most equitable outcome Company Name September 2010 to October 2013 FORECLOSURE PROCESSOR PARALEGAL City State Diligently reviewed the specialty loan portfolio for compliance with all reporting requirements Communicated regularly with management regarding portfolio performance and new loan transaction quality Maintained confidentiality of bank records and client information Scanned and filed forms reports correspondence and receipts Entered information into computer databases Reviewed files to check for complete and accurate information Examined Deeds of Trust to determine the grantor grantee trustee and loan amount Coordinated with multiple departments regarding responsive documents and document retention Researched bankruptcy loan files to confirm federal guideline compliance Supported a team of three attorneys with generating and filing of pleadings motions and various court documents Company Name February 2008 to May 2008 TOEFLTESL INSTRUCTOR City State Developed interesting course plans to meet academic intellectual and social needs of students Developed and implemented interesting and interactive learning mediums to increase student understanding of course materials Performed student background reviews to develop tailored lessons based on student needs Developed administered and corrected tests and quizzes in a timely manner Combined discipline plan with effective measures', 0.8498380726099971, 0.1501619273900029),\n", - " ('18187364.pdf', '35_2', 'State DBA for telesales signature verification and electronic payment systems Participated in offsite disaster recovery exercises Reviewed schema tuned queries and managed change control process Developed Cost Based SQL Standards and trained development staff on SQL tuning Provided database design consultation to other projects Developed database installation and administration guidelines Senior Database Administrator June 1997 to December 1997 Company Name i14 City State Converted document management system from Sybase to Oracle Mentored and trained Oracle database administrators at client sites Monitored and tuned Oracle system and applications to prevent resource shortages and shorten the execution time of longrunning queries Conducted training in database concepts and SQL Database Administrator September 1996 to June 1997 Company Name i14 City State Implemented and maintained critical high volume online and Internet server Oracle databases in UNIX environment Performed performance monitoring capacity planning and application tuning Worked closely with engineering consulting firm to trouble shoot database and applications optimize system performance ensure data integrity and increase system reliability Wrote extensive SQL and PLSQL programs to manage data and create ad hoc reports Developed implemented and enforced Oracle design and usage standards Associate Computer ProgrammerAnalyst June 1991 to September 1996 Company Name i14 City State Technical lead responsible for Pavement and Bridge Management Systems development and production Oracle databases operating in clientserver environment Prepared EDP sections of consulting contracts and budgets Managing analyst for Pavement and Bridge Maintenance Systems jointly developed by Rensselaer Polytechnic Institute and the Thruway Authority Developed and maintained data standards and agency data dictionary system Education Master of Science Management College of Saint Rose 14 City State Management Bachelor of Arts Music History City State Music History Skills account management ad analyst application development ASM agency audit reports auditing backup budgets c Capability Maturity Model CMM capacity planning clientserver COGNOS concept conferences consultation consulting contracts client data dictionary system database and applications database administration DBA databases Database database design Designing disaster recovery document management due diligence government regulations IBM Information Security Information Systems law Managing meetings mentoring access Money Windows NT modeling Enterprise operating systems Oracle Enterprise Manager Oracle Oracle database PLSQL page PeopleSoft policies procurement Oracle RDBMS Risk Assessment scanning servers scripts Scripting software development SQL SQLLoader statistics Sybase Systems development Tivoli training programs troubleshooting UNIX UNIX shell scripts upgrade workshops', 0.8442204413166462, 0.1557795586833538),\n", - " ('18187364.pdf', '35_1', 'Soft upgrade and AntiMoney Laundering projects Responsible for operational aspects of Oracle database administration activities including capacity planning installation and configuration of the Oracle RDBMS Grid Control and ASM software patches and supporting products backup recovery database tuning monitoring and troubleshooting utilizing TKPROF OEM STATSPACK DBArtisan Tivoli and custom SQL PLSQL and UNIX shell scripts Plan and manage multilocation disaster recovery exercises Provide operational 24X7 support of all corporate Oracle systems 341 databases 65 servers 5 versions of Oracle and 5 operating systems Developed and implemented procedures that reduced inhouse database problem tickets by 60 job failures by 80 and oncall support issues by 80 Created enterprise wide capacity planning troubleshooting and performance monitoring models Coordinated and supported application development testing and performance improvement efforts including data model revisions SQL tuning and client configurations Instituted a series of workshops classes and training programs for developers to expand their knowledge and understanding of SQL Oracle and data security This group is now selfsufficient Performed blocklevel data recovery that Oracle Corporation said was not possible saving critical business data and minimizing impact to business functions Database Manager February 2000 to April 2002 Company Name i14 City State Created and supported multiinstance spatial environments for internet startup company Gathered user requirements and designed and built logical and physical database structures Managed Unix server farm to ensure proper sizing organization and recoverability Wrote PLSQL SQLLoader and custom routines to load and integrate data from various outside sources and to enforce data security reliability and integrity Monitored shared system resources and recommend improvements to application development staff Wrote databasemonitoring scripts used to page DBA in the event of database problems Automated DBA functions for table restructuring statistics space management and backup Senior Database Administrator January 1999 to February 2000 Company Name i14 City State Technical liaison and support manager for international leasing company Traveled abroad as needed Participated in due diligence audits of takeover candidate companies Wrote Oracle installation and configuration standards for Windows NT and UNIX Created DBA practice lab and developed practice lab exercises for other DBA staff to learn backup and recovery software Worked closely with various vendors and development groups to improve application reliability and performance Developed a Capability Maturity Model and created CMM training program for database administration Provided 24X7support of international commercial leasing applications System Staff SpecialistDatabase Administrator December 1997 to January 1999 Company Name i14 City', 0.8387444263775664, 0.1612555736224336)]" - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import os\n", - "import json\n", - "from dotenv import load_dotenv\n", - "\n", - "def vector_search_sql(query, num_results=5):\n", - " # Load environment variables from .env file\n", - " load_dotenv()\n", - "\n", - " # Use the get_mssql_connection function to get the connection\n", - " conn = get_mssql_connection()\n", - "\n", - " # Create a cursor object\n", - " cursor = conn.cursor()\n", - "\n", - " # Generate the query embedding for the user's search query\n", - " user_query_embedding = get_embedding(query)\n", - "\n", - " # SQL similarity search via VECTOR_DISTANCE. The query vector goes over the\n", - " # wire as JSON text, then CAST twice: NVARCHAR(MAX) first (mssql-python\n", - " # binds Python strings as ntext), then VECTOR(1536).\n", - " sql_similarity_search = \"\"\"\n", - " SELECT TOP(?) filename, chunkid, chunk,\n", - " 1 - VECTOR_DISTANCE('cosine', CAST(CAST(? AS NVARCHAR(MAX)) AS VECTOR(1536)), embedding) AS similarity_score,\n", - " VECTOR_DISTANCE('cosine', CAST(CAST(? AS NVARCHAR(MAX)) AS VECTOR(1536)), embedding) AS distance_score\n", - " FROM dbo.resumedocs\n", - " ORDER BY distance_score\n", - " \"\"\"\n", - "\n", - " cursor.execute(sql_similarity_search,\n", - " num_results,\n", - " json.dumps(user_query_embedding),\n", - " json.dumps(user_query_embedding))\n", - " results = cursor.fetchall()\n", - "\n", - " # Close the database connection\n", - " conn.close()\n", - " return results\n", - "\n", - "# example usage\n", - "vector_search_sql(\"database administrator\", num_results=3)\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": { - "azdata_cell_guid": "1e194f3c-6a7a-4f16-95ec-05f60a3770a4", - "language": "python" - }, - "source": [ - "# **Part 4 : Use embeddings retrieved from a Azure SQL vector database to augment LLM generation**\n", - "\n", - "Lets create a helper function to feed prompts into the [Completions model](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#gpt-4) & create interactive loop where you can pose questions to the model and receive information grounded in your data.\n", - "\n", - "The function `generate_completion` is defined to help ground the gpt-4o model with prompts and system instructions.  \n", - "Note that we are passing the results of the `vector_search_sql` we defined earlier to the model and we define the system prompt . \n", - "We are using gpt-4o model here. \n", - "\n", - "You can get more information on using Azure Open AI GPT chat models [here](https://learn.microsoft.com/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cpython-new&pivots=programming-language-python)" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": { - "azdata_cell_guid": "2865a0d0-2ee3-4d0a-8ee8-89075ab541bc", - "language": "python" - }, - "outputs": [], - "source": [ - "import os\n", - "from dotenv import load_dotenv\n", - "from openai import AzureOpenAI\n", - "\n", - "# Load environment variables from a .env file\n", - "load_dotenv()\n", - "\n", - "# Use environment variables for the API key and endpoint\n", - "api_key = os.getenv(\"AZOPENAI_API_KEY\")\n", - "azure_endpoint = os.getenv(\"AZOPENAI_ENDPOINT\")\n", - "chat_model = os.getenv(\"AZOPENAI_CHAT_MODEL_DEPLOYMENT_NAME\")\n", - "\n", - "# Create a chat completion request\n", - "client = AzureOpenAI(\n", - " api_key=api_key,\n", - " api_version=\"2023-05-15\",\n", - " azure_endpoint=azure_endpoint\n", - ")\n", - "\n", - "def generate_completion(search_results, user_input):\n", - " system_prompt = '''\n", - "You are an intelligent & funny assistant who will exclusively answer based on the data provided in the `search_results`:\n", - "- Use the information from `search_results` to generate your top 3 responses. If the data is not a perfect match for the user's query, use your best judgment to provide helpful suggestions and include the following format:\n", - " File: {filename}\n", - " Chunk ID: {chunkid}\n", - " Similarity Score: {similarity_score}\n", - " Add a small snippet from the Relevant Text: {chunktext}\n", - " Do not use the entire chunk\n", - "- Avoid any other external data sources.\n", - "- Add a summary about why the candidate maybe a goodfit even if exact skills and the role being hired for are not matching , at the end of the recommendations. Ensure you call out which skills match the description and which ones are missing. If the candidate doesnt have prior experience for the hiring role which we may need to pay extra attention to during the interview process.\n", - "- Add a Microsoft related interesting fact about the technology that was searched \n", - "'''\n", - "\n", - " messages = [{\"role\": \"system\", \"content\": system_prompt}]\n", - " \n", - " # Create an empty list to store the results\n", - " result_list = []\n", - "\n", - " # Iterate through the search results and append relevant information to the list\n", - " for result in search_results:\n", - " filename = result # Assuming filename is the first column\n", - " chunkid = result\n", - " chunktext = result\n", - " similarity_score = result # Assuming similarity_score is the third column\n", - " \n", - " # Append the relevant information as a dictionary to the result_list\n", - " result_list.append({\n", - " \"filename\": filename,\n", - " \"chunkid\": chunkid,\n", - " \"chunktext\": chunktext,\n", - " \"similarity_score\": similarity_score\n", - " })\n", - "\n", - " # Print the result list\n", - " #print(result_list)\n", - " \n", - " messages.append({\"role\": \"system\", \"content\": f\"{result_list}\"})\n", - " messages.append({\"role\": \"user\", \"content\": user_input})\n", - " response = client.chat.completions.create(model=chat_model, messages=messages, temperature=0) \n", - "\n", - " return response.dict()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "azdata_cell_guid": "d69ab285-0cc9-4596-95f2-688d0c324f6b", - "language": "python", - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "*** What Role are you hiring for? And What skills are you looking for? Ask me & I can help you find a candidate :) Type 'end' to end the session.\n", - "\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "User asked: We are hiring a Product Manager in the Microsoft Azure Database Migration team. Candidate should have good knowledge on SQL and any other databases like Oracle, PostgreSQL etc. It would be beneficial if they have had cloud experience . We seek strong handson skills in migration projects \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "AI's response:\n", - "Here are the top 3 candidates based on the provided search results:\n", - "\n", - "### Candidate 1:\n", - "**File:** 18067556.pdf \n", - "**Chunk ID:** 32_6 \n", - "**Similarity Score:** 0.8163 \n", - "**Relevant Text Snippet:** \n", - "\"Strong knowledge of AWS, Azure, Cisco Switch Management, VMWare, HyperV, RDP, Automation Anywhere, Active Directory, and hardware and software administration for IOS, Android, Windows, Red Hat Linux, RF devices... Programming Databases SQL, SSRS, SSIS, SSAS, VBA, DAX, HTML, CSS, VBA, VB.NET, R, Powershell, Python, Oracle... Business Intelligence Packages PowerBI, Qlik, Qlik View, QlikSense, SiSense, Tableau, Datorama, Yellowfin, Crystal, SSRS.\"\n", - "\n", - "**Summary:** This candidate has extensive experience with SQL, Oracle, and Azure, which aligns well with the requirements for the Product Manager role in the Azure Database Migration team. They also have experience with various business intelligence tools and cloud platforms, including AWS and Azure. However, specific hands-on migration project experience is not explicitly mentioned and should be explored during the interview.\n", - "\n", - "### Candidate 2:\n", - "**File:** 29051656.pdf \n", - "**Chunk ID:** 81_0 \n", - "**Similarity Score:** 0.8146 \n", - "**Relevant Text Snippet:** \n", - "\"An organized DBA professional with over 6 years hands-on experience supporting Oracle databases, SQL Server databases, and AWS infrastructure... Migrated databases from on-premise to AWS using Database migration services... Launched and maintained RDS and EC2 instances in AWS.\"\n", - "\n", - "**Summary:** This candidate has significant hands-on experience with database migration projects, particularly with Oracle and SQL Server databases. They also have cloud experience with AWS, which is beneficial. While Azure experience is not explicitly mentioned, their strong background in database migration and cloud infrastructure makes them a strong candidate for the role.\n", - "\n", - "### Candidate 3:\n", - "**File:** 26746496.pdf \n", - "**Chunk ID:** 67_0 \n", - "**Similarity Score:** 0.8115 \n", - "**Relevant Text Snippet:** \n", - "\"Software Engineer with 2 years in Web Developer specializing in front end development... Good experience in writing Class Library using C#, LINQ to SQL queries in Database Access layer to interface with SQL Database... Worked extensively with .NET Server Controls, Web User Controls, Data Grid Web Control, Form Validation Controls, and created Custom controls.\"\n", - "\n", - "**Summary:** This candidate has experience with SQL and .NET technologies, which are relevant to the role. However, their experience seems more focused on web development and front-end technologies rather than database migration. They have some cloud experience but lack explicit mention of hands-on migration projects, which should be further investigated during the interview.\n", - "\n", - "### Summary:\n", - "- **Candidate 1**: Strong in SQL, Oracle, and Azure with broad technical skills. Missing explicit hands-on migration project experience.\n", - "- **Candidate 2**: Extensive hands-on migration experience with Oracle and SQL Server, strong cloud experience with AWS. Missing explicit Azure experience.\n", - "- **Candidate 3**: Good SQL and .NET experience, but more focused on web development. Missing explicit hands-on migration project experience and Azure experience.\n", - "\n", - "### Microsoft Fact:\n", - "Did you know that Microsoft Azure's Database Migration Service (DMS) supports seamless migrations from multiple database sources, including SQL Server, Oracle, and PostgreSQL, to Azure with minimal downtime? This service is designed to simplify and accelerate the migration process, making it easier for organizations to move their databases to the cloud.\n" - ] - } - ], - "source": [ - "# Create a loop of user input and model output to perform Q&A on the PDF's that are now chunked and stored in the SQL DB with embeddings\n", - "#\n", - "# PLEASE NOTE: An input box will be displayed for the user to enter a question/query at the top of the scree.\n", - "# The model will then provide a response based on the data stored in the SQL DB.\n", - "# Type 'end' to end the session.\n", - "#\n", - "print(\"*** What Role are you hiring for? And What skills are you looking for? Ask me & I can help you find a candidate :) Type 'end' to end the session.\\n\")\n", - "\n", - "while True:\n", - " user_input = input(\"User prompt: \")\n", - " if user_input.lower() == \"end\":\n", - " break\n", - "\n", - " # Print the user's question\n", - " print(f\"\\nUser asked: {user_input}\")\n", - "\n", - " \n", - " # Assuming vector_search_sql and generate_completion are defined functions that work correctly\n", - " search_results = vector_search_sql(user_input)\n", - " completions_results = generate_completion(search_results, user_input)\n", - "\n", - " # Print the model's response\n", - " print(\"\\nAI's response:\")\n", - " print(completions_results['choices'][0]['message']['content'])\n", - "\n", - "# The loop will continue until the user types 'end'\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "507219d1-d713-4c41-86d5-e938bf69627c", + "language": "sql" + }, + "source": [ + "# Leveraging Azure SQL DB’s Native Vector Capabilities for Enhanced Resume Matching with Azure Document Intelligence and RAG\n", + "\n", + "In this tutorial, we will explore how to leverage Azure SQL DB’s new vector data type to store embeddings and perform similarity searches using built-in vector functions, enabling advanced resume matching to identify the most suitable candidates. \n", + "\n", + "By extracting and chunking content from PDF resumes using Azure Document Intelligence, generating embeddings with Azure OpenAI, and storing these embeddings in Azure SQL DB, we can perform sophisticated vector similarity searches and retrieval-augmented generation (RAG) to identify the most suitable candidates based on their resumes.\n", + "\n", + "### **Tutorial Overview**\n", + "\n", + "- This Python notebook will teach you to:\n", + " 1. **Chunk PDF Resumes**: Use **`Azure Document Intelligence`** to extract and chunk content from PDF resumes.\n", + " 2. **Create Embeddings**: Generate embeddings from the chunked content using the **`Azure OpenAI API`**.\n", + " 3. **Vector Database Utilization**: Store embeddings in **`Azure SQL DB`** utilizing the **`new Vector Data Type`** and perform similarity searches using built-in vector functions to find the most suitable candidates.\n", + " 4. **LLM Generation Augmentation**: Enhance language model generation with embeddings from a vector database. In this case, we use the embeddings to inform a GPT-4 chat model, enabling it to provide rich, context-aware answers about candidates based on their resumes\n", + "\n", + "## Dataset\n", + "\n", + "We use a sample dataset from [Kaggle](https://www.kaggle.com/datasets/snehaanbhawal/resume-dataset) containing PDF resumes for this tutorial. For the purpose of this tutorial we will use 120 resumes from the **Information-Technology** folder\n", + "\n", + "## Prerequisites\n", + "\n", + "- **Azure Subscription**: [Create one for free](https://azure.microsoft.com/free/cognitive-services?azure-portal=true)\n", + "- **Azure SQL Database**: [Set up your database for free](https://learn.microsoft.com/azure/azure-sql/database/free-offer?view=azuresql)\n", + "- **Azure Document Intelligence** [Create a FreeAzure Doc Intelligence resource](https:/learn.microsoft.com/azure/ai-services/document-intelligence/create-document-intelligence-resource?view=doc-intel-4.0.0)\n", + "- **Azure Data Studio**: Download [here](https://azure.microsoft.com/products/data-studio) to manage your Azure SQL database and [execute the notebook](https://learn.microsoft.com/azure-data-studio/notebooks/notebooks-python-kernel)\n", + "\n", + "## Additional Requirements for Embedding Generation\n", + "\n", + "- **Azure OpenAI Access**: Apply for access in the desired Azure subscription at [https://aka.ms/oai/access](https://aka.ms/oai/access)\n", + "- **Azure OpenAI Resource**: Deploy an embeddings model (e.g., `text-embedding-small` or `text-embedding-ada-002`) and a `GPT-4.0` model for chat completion. Refer to the [resource deployment guide](https://learn.microsoft.com/azure/ai-services/openai/how-to/create-resource)\n", + "- **Python**: Version 3.7.1 or later from Python.org. (Sample has been tested with Python 3.11)\n", + "- **Python Libraries**: Install the required libraries openai, num2words, matplotlib, plotly, scipy, scikit-learn, pandas, tiktoken, and mssql-python.\n", + "- **Jupyter Notebooks**: Use within [Azure Data Studio](https://learn.microsoft.com/en-us/azure-data-studio/notebooks/notebooks-guidance) or Visual Studio Code .\n", + "\n", + "Code snippets are adapted from the [Azure OpenAI Service embeddings Tutorial](https://learn.microsoft.com/en-us/azure/ai-services/openai/tutorials/embeddings?tabs=python-new%2Ccommand-line&pivots=programming-language-python)\n", + "\n", + "## Getting Started\n", + "\n", + "1. **Database Setup**: Execute SQL commands from the `createtable.sql` script to create the necessary table in your database.\n", + "2. **Model Deployment**: Deploy an embeddings model (`text-embedding-small` or `text-embedding-ada-002`) and a `GPT-4` model for chat completion. Note the 2 models deployment names for later use.\n", + "\n", + "![Deployed OpenAI Models](../Assets/modeldeployment.png)\n", + "\n", + "3. **Connection String**: Find your Azure SQL DB connection string in the Azure portal under your database settings.\n", + "4. **Configuration**: Populate the `.env` file with your SQL server connection details , Azure OpenAI key and endpoint, Azure Document Intelligence key and endpoint values.\n", + "\n", + "You can retrieve the Azure OpenAI _endpoint_ and _key_:\n", + "\n", + "![Azure OpenAI Endpoint and Key](../Assets/endpoint.png)\n", + "\n", + "You can [retrieve](https://learn.microsoft.com/azure/ai-services/document-intelligence/create-document-intelligence-resource?view=doc-intel-4.0.0#get-endpoint-url-and-keys) the Document Intelligence _endpoint_ and _key_:\n", + "\n", + "![Azure Document Intelligence Endpoint and Key](../Assets/docintelendpoint.png)\n", + "\n", + "## Running the Notebook\n", + "\n", + "To [execute the notebook](https://learn.microsoft.com/azure-data-studio/notebooks/notebooks-python-kernel), connect to your Azure SQL database using Azure Data Studio, which can be downloaded [here](https://azure.microsoft.com/products/data-studio)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "fe37c601-5918-4055-badc-6c0ba90c68ce", + "language": "python" + }, + "outputs": [], + "source": [ + "#Setup the python libraries required for this notebook\n", + "#Please ensure that you navigate to the directory containing the `requirements.txt` file in your terminal\n", + "%pip install -r requirements.txt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "4c29709e-1c3a-495d-83ec-05737e220847", + "language": "python" + }, + "outputs": [], + "source": [ + "#Load the env details\n", + "from dotenv import load_dotenv\n", + "load_dotenv()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "4b543f05-9036-4887-8737-09aa9f865ec2", + "language": "python" + }, + "source": [ + "# **PART 1: Extracting and Chunking Text from PDF Resumes using Azure Document Intelligence**\n", + "\n", + "Create an instance of the [DocumentAnalysisClient](https://learn.microsoft.com/azure/ai-services/document-intelligence/create-document-intelligence-resource?view=doc-intel-4.0.0#get-endpoint-url-and-keys) using the endpoint and API key. \n", + "\n", + "[Azure Document Intelligence](https://learn.microsoft.com/azure/ai-services/document-intelligence/?view=doc-intel-4.0.0_)(previously known as Form Recognizer) is a Azure cloud service that uses machine learning to analyze text and structured data from your documents. This client will be used to send requests to the [Azure Document Intelligence](https://learn.microsoft.com/python/api/overview/azure/ai-formrecognizer-readme?view=azure-python) service and receive responses containing the extracted text from the PDF resumes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "b20cc66a-50ce-4486-b275-d4683f4ba545", + "language": "python" + }, + "outputs": [], + "source": [ + "import os\n", + "import re\n", + "from azure.ai.formrecognizer import DocumentAnalysisClient\n", + "from azure.core.credentials import AzureKeyCredential\n", + "\n", + "# Load environment variables\n", + "endpoint = os.getenv(\"AZUREDOCINTELLIGENCE_ENDPOINT\")\n", + "api_key = os.getenv(\"AZUREDOCINTELLIGENCE_API_KEY\")\n", + "\n", + "# Create a DocumentAnalysisClient\n", + "document_analysis_client = DocumentAnalysisClient(\n", + " endpoint=endpoint,\n", + " credential=AzureKeyCredential(api_key)\n", + ")\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "1ecc04b0-1a5f-4d48-819d-2cc06a070062", + "language": "python" + }, + "source": [ + "### **Analyze input documents using prebuilt model in Azure Document Intelligence**\n", + "\n", + "- DocumentAnalysisClient provides operations for analyzing input documents using prebuilt and custom models through the `begin_analyze_document` and `begin_analyze_document_from_url` APIs. In this tutorial we are using the [prebuilt-layout](https://learn.microsoft.com/python/api/overview/azure/ai-formrecognizer-readme?view=azure-python#using-prebuilt-models)\n", + " \n", + "\n", + "### **Split text into chunks of 500 tokens**\n", + "\n", + "- When faced with content that exceeds the embedding limit, we usually also chunk the content into smaller pieces and then embed those one at a time. Here we will use [tiktoken](https://github.com/openai/tiktoken?tab=readme-ov-file) to chunk the extracted text into token sizes of 500, as we will later pass the extracted chunks to to the `text-embedding-small` model for [generating text embeddings](https://learn.microsoft.com/azure/ai-services/openai/tutorials/embeddings?tabs=python-new%2Ccommand-line&pivots=programming-language-python) as this has a model input token limit of 8192.\n", + "\n", + "**Note**: You need to provide the location of the folder where the PDF files reside in the below script." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "0355f92c-0546-4eac-aea8-b55d8bbef194", + "language": "python" + }, + "outputs": [], + "source": [ + "import os\n", + "import re\n", + "import pandas as pd\n", + "import tiktoken\n", + "\n", + "# Path to the directory containing PDF files\n", + "folder_path = os.path.join(os.getcwd(),'C:\\\\vectortest\\\\resumedata\\\\data\\\\data\\\\INFORMATION-TECHNOLOGY')\n", + "\n", + "def get_pdf_files(folder_path):\n", + " for path, subdirs, files in os.walk(folder_path):\n", + " for name in files:\n", + " if (name.endswith(\".pdf\")):\n", + " yield os.path.join(path, name)\n", + "\n", + "# Function to read PDF files and extract text using Azure AI Document Intelligence\n", + "def extract_text_from_pdf(pdf_path):\n", + " with open(pdf_path, \"rb\") as f:\n", + " poller = document_analysis_client.begin_analyze_document(\"prebuilt-layout\", document=f)\n", + " result = poller.result()\n", + " text = \"\"\n", + " for page in result.pages:\n", + " for line in page.lines:\n", + " text += line.content + \" \"\n", + " return text\n", + "\n", + "# Function to clean text and remove special characters\n", + "def clean_text(text):\n", + " text = re.sub(r'\\s+', ' ', text) # Remove extra whitespace\n", + " text = re.sub(r'[^a-zA-Z0-9\\s]', '', text) # Remove special characters\n", + " return text\n", + "\n", + "# Function to split text into chunks of 500 tokens\n", + "def split_text_into_token_chunks(text, max_tokens=500):\n", + " tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n", + " tokens = tokenizer.encode(text)\n", + " chunks = []\n", + " \n", + " for i in range(0, len(tokens), max_tokens):\n", + " chunk_tokens = tokens[i:i + max_tokens]\n", + " chunk_text = tokenizer.decode(chunk_tokens)\n", + " chunks.append(chunk_text)\n", + " \n", + " return chunks\n", + "\n", + "# Count the number of PDF files in the directory\n", + "pdf_files = [f for f in get_pdf_files(folder_path)]\n", + "num_files = len(pdf_files)\n", + "print(f\"Number of PDF files in the directory: {num_files}\")\n", + "\n", + "#Extract the file name\n", + "for pdf_file in pdf_files:\n", + " file_name = os.path.basename(pdf_file)\n", + "\n", + "# Create a DataFrame to store the chunks\n", + "data = []\n", + "\n", + "for file_id, pdf_file in enumerate(pdf_files):\n", + " print(f\"Processing file {file_id + 1}/{num_files}: {file_name}\")\n", + " pdf_path = os.path.join(folder_path, pdf_file)\n", + " text = extract_text_from_pdf(pdf_path)\n", + " cleaned_text = clean_text(text)\n", + " chunks = split_text_into_token_chunks(cleaned_text)\n", + " \n", + " print(f\"Number of chunks for file {file_name}: {len(chunks)}\")\n", + " \n", + " for chunk_id, chunk in enumerate(chunks):\n", + " chunk_text = chunk.strip() if chunk.strip() else \"NULL\"\n", + " unique_chunk_id = f\"{file_id}_{chunk_id}\"\n", + " print(f\"File: {file_name}, Chunk ID: {chunk_id}, Unique Chunk ID: {unique_chunk_id}, Chunk Length: {len(chunk_text)}, Chunk Text: {chunk_text[:50]}...\") # Print first 50 characters of chunk text\n", + " data.append({\n", + " \"file_name\": file_name,\n", + " \"chunk_id\": chunk_id,\n", + " \"chunk_text\": chunk_text,\n", + " \"unique_chunk_id\": unique_chunk_id\n", + " })\n", + "\n", + "df = pd.DataFrame(data)\n", + "df.head(3)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "3c0b7dba-c798-40d1-ac92-80288633497a", + "language": "python" + }, + "outputs": [], + "source": [ + "#read the top5 rows of the dataframe\n", + "df.head(5)\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "655a8389-1018-4e80-a39d-85187cf3c46f", + "language": "python" + }, + "source": [ + "### **Tokenization vs. Character Length (OPTIONAL)**\n", + "\n", + "In this section, we will explore the difference between the character length of a text chunk and its tokenized representation. Character length simply counts the number of characters in a text, while tokenization breaks the text into meaningful units called tokens.\n", + "\n", + "Character Length First, let’s add a new column to our DataFrame to view the length of each chunk in terms of characters: Here, chunk\\_length represents the number of characters in each chunk." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "5e171928-764a-4d61-899a-83a8e0c3ac79", + "language": "python" + }, + "outputs": [], + "source": [ + "# Add a new column 'chunk_length' to the DataFrame to view the length of each chunk\n", + "df['chunk_length'] = df['chunk_text'].apply(len)\n", + "\n", + "# Display the first few rows of the DataFrame with the new column\n", + "print(df[['file_name', 'chunk_id', 'chunk_length']].head(5))\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "9b87f84b-149c-4eb5-b9f0-c1b55f6d606c", + "language": "python" + }, + "source": [ + "### Tokenization\n", + "To understand how text ultimately is tokenized, it can be helpful to run the below code: \n", + "\n", + "- We use the tiktoken library to tokenize the text. Tokenization breaks the text into smaller units, which can be words, subwords, or characters, depending on the tokenizer used. You can see that in some cases an entire word is represented with a single token whereas in others parts of words are split across multiple tokens. \n", + "\n", + "- If you then check the length of the decode variable, you'll find it matches 500 our specified token number. It is simply a way of making sure none of the data we pass to the model for tokenization and embedding exceeds the input token limit of 8,192\n", + "\n", + "- When we pass the documents to the embeddings model, it will break the documents into tokens similar (though not necessarily identical) to the examples below and then convert the tokens to a series of floating point numbers that will be accessible via vector search" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "28188280-2bef-414a-a663-a017c944bc19", + "language": "python" + }, + "outputs": [], + "source": [ + "import tiktoken\n", + "tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n", + "sample_encode = tokenizer.encode(df.chunk_text[0]) \n", + "decode = tokenizer.decode_tokens_bytes(sample_encode)\n", + "decode\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "31d80228-e140-413a-b1e3-c320a42b1f6c", + "language": "python" + }, + "outputs": [], + "source": [ + "len(decode)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "ac19ee49-763a-4e2e-8c31-9ce1ba77bbe4" + }, + "source": [ + "# **PART 2 : Generating Embeddings for Text Chunks using Azure Open AI**\n", + "\n", + "- After extracting and chunking the text from PDF resumes, we will generate embeddings for each chunk. These embeddings are numerical representations of the text that capture its semantic meaning. By creating embeddings for the text chunks, we can perform advanced similarity searches and enhance language model generation.\n", + "\n", + "- We will use the Azure OpenAI API to generate these embeddings. The `get_embedding` function defined below takes a piece of text as input and returns its embedding using the `text-embedding-small` model\n", + "\n", + "- Ensure the Environment Variables are set correctly in the .env file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "3aeda057-d0c0-40cd-bdcf-723abfed94e2", + "language": "python" + }, + "outputs": [], + "source": [ + "import os\n", + "import requests\n", + "from num2words import num2words\n", + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "from openai import AzureOpenAI\n", + "\n", + "# Specify your model name\n", + "openai_embedding_model = os.getenv(\"AZOPENAI_EMBEDDING_MODEL_DEPLOYMENT_NAME\")\n", + "\n", + "# Assuming openai_url and openai_key are your environment variables\n", + "openai_url = os.getenv(\"AZOPENAI_ENDPOINT\") + \"openai/deployments/\" + openai_embedding_model + \"/embeddings?api-version=2023-05-15\"\n", + "openai_key = os.getenv(\"AZOPENAI_API_KEY\")\n", + "\n", + "def get_embedding(text):\n", + " \"\"\"\n", + " Get sentence embedding using the Azure OpenAI text-embedding-small model.\n", + "\n", + " Args:\n", + " text (str): Text to embed.\n", + "\n", + " Returns:\n", + " list: A list containing the embedding.\n", + " \"\"\"\n", + " response = requests.post(openai_url,\n", + " headers={\"api-key\": openai_key, \"Content-Type\": \"application/json\"},\n", + " json={\"input\": [text]} # Embed the extracted chunk\n", + " )\n", + " \n", + " if response.status_code == 200:\n", + " response_json = response.json()\n", + " embedding = json.loads(str(response_json['data'][0]['embedding']))\n", + " return embedding\n", + " else:\n", + " return None\n", + "\n", + "# Example usage\n", + "all_filenames = []\n", + "all_chunkids = []\n", + "all_chunks = []\n", + "all_embeddings = []\n", + "\n", + "# Assuming df is already defined with the required columns\n", + "for index, row in df.iterrows():\n", + " filename = row['file_name']\n", + " chunkid = row['unique_chunk_id']\n", + " chunk = row['chunk_text']\n", + " embedding = get_embedding(chunk)\n", + " \n", + " if embedding is not None:\n", + " all_filenames.append(filename)\n", + " all_chunkids.append(chunkid)\n", + " all_chunks.append(chunk)\n", + " all_embeddings.append(embedding)\n", + " \n", + " if (index + 1) % 50 == 0: # Print progress every 50 rows\n", + " print(f\"Completed {index + 1} rows\")\n", + "\n", + "# Create a new DataFrame with the results\n", + "result_df = pd.DataFrame({\n", + " 'filename': all_filenames,\n", + " 'chunkid': all_chunkids,\n", + " 'chunk': all_chunks,\n", + " 'embedding': all_embeddings\n", + "})\n", + "\n", + "print(result_df.head(5)) # Display the first few rows of the dataframe\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "04d42351-41ec-4ce9-9778-fe4ea2ecb8a2" + }, + "source": [ + "# **PART 3 : Using Azure SQL DB as a Vector Database to store and query embeddings**\n", + "\n", + "### **Load the embeddings into the Vector Database : Azure SQL DB**\n", + "\n", + "First let us define a function to connect to Azure SQLDB" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "930a63bc-4c08-4205-b152-b1ad5c82057a", + "language": "python" + }, + "outputs": [], + "source": [ + "# Connect to Azure SQL using mssql-python — the first-party Microsoft Python\n", + "# driver. Native Entra ID support (no manual token struct) and no ODBC driver\n", + "# install required. See https://github.com/microsoft/mssql-python\n", + "#\n", + "# pip install mssql-python\n", + "#\n", + "# If you prefer pyodbc, the old pyodbc + struct-token pattern still works —\n", + "# see the git history of this file for the previous version.\n", + "\n", + "import os\n", + "from dotenv import load_dotenv\n", + "import mssql_python\n", + "\n", + "# Load environment variables from .env file\n", + "load_dotenv()\n", + "\n", + "def get_mssql_connection():\n", + " # Retrieve the connection string from the environment variables\n", + " entra_connection_string = os.getenv('ENTRA_CONNECTION_STRING')\n", + " sql_connection_string = os.getenv('SQL_CONNECTION_STRING')\n", + "\n", + " if entra_connection_string:\n", + " # Entra ID (passwordless) — mssql-python handles the token exchange for\n", + " # you via `authentication='ActiveDirectoryDefault'`, which chains\n", + " # DefaultAzureCredential (Azure CLI, VS Code, managed identity, etc.).\n", + " return mssql_python.connect(entra_connection_string,\n", + " authentication='ActiveDirectoryDefault')\n", + " elif sql_connection_string:\n", + " # SQL Authentication — connection string carries user + password\n", + " return mssql_python.connect(sql_connection_string)\n", + " else:\n", + " raise ValueError(\"No valid connection string found in the environment variables.\")\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "e929c112-65d4-46f1-a9a2-7c9434b2d7cb", + "language": "python" + }, + "source": [ + "### **Insert embeddings into the native 'Vector' Data Type**\n", + "\n", + "We will insert our vectors into the SQL Table now. Azure SQL DB now has a dedicated, native, data type for storing vectors: the `vector` data type. Read about the preview [here](https://devblogs.microsoft.com/azure-sql/eap-for-vector-support-refresh-introducing-vector-type)\n", + "\n", + "The table embeddings has a column called vector which is vector(1536) type. Ensure you have created the table using the script `CreateTable.sql` before running the below code." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "680259d9-77ce-4b63-b412-9bea33bb0f43", + "language": "python" + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "# Retrieve the connection string from the function get_mssql_connection()\n", + "conn = get_mssql_connection()\n", + "\n", + "# Create a cursor object\n", + "cursor = conn.cursor()\n", + "\n", + "# Loop through the DataFrame rows and insert them into the table.\n", + "#\n", + "# The embedding is sent as a JSON string over the wire, then cast twice:\n", + "# NVARCHAR(MAX) first (mssql-python binds Python strings as ntext, which\n", + "# CAST cannot convert directly to VECTOR), then VECTOR(1536).\n", + "for index, row in result_df.iterrows():\n", + " chunkid = row['chunkid']\n", + " filename = row['filename']\n", + " chunk = row['chunk']\n", + " embedding = row['embedding']\n", + "\n", + " query = \"\"\"\n", + " INSERT INTO resumedocs (chunkid, filename, chunk, embedding)\n", + " VALUES (?, ?, ?, CAST(CAST(? AS NVARCHAR(MAX)) AS VECTOR(1536)))\n", + " \"\"\"\n", + " cursor.execute(query, chunkid, filename, chunk, json.dumps(embedding))\n", + "\n", + "# Commit the changes\n", + "conn.commit()\n", + "\n", + "print(\"Data inserted successfully into the 'resumedocs' table.\")\n", + "\n", + "# Close the connection\n", + "conn.close()\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "ede638b9-d681-4cb9-9ab8-9651e3099a36", + "language": "python" + }, + "source": [ + "Let's take a look at the data in the Resume Docs table:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "88d1a90e-e93c-426e-934d-fb1a47dfd900", + "language": "python" + }, + "outputs": [], + "source": [ + "from prettytable import PrettyTable\n", + "import pandas as pd\n", + "\n", + "# Load environment variables from .env file\n", + "load_dotenv()\n", + "\n", + "# Retrieve the connection using the mssql-python-backed helper\n", + "conn = get_mssql_connection()\n", + "\n", + "# Create a cursor object\n", + "cursor = conn.cursor()\n", + "\n", + "# Use placeholders for the parameters in the SQL query\n", + "query = \"SELECT TOP(10) filename, chunkid, chunk, CAST(embedding AS NVARCHAR(MAX)) as embedding FROM dbo.resumedocs ORDER BY Id\"\n", + "\n", + "# Execute the query with the parameters\n", + "cursor.execute(query)\n", + "queryresults = cursor.fetchall()\n", + "\n", + "# Get column names from cursor.description\n", + "column_names = [column[0] for column in cursor.description]\n", + "\n", + "# Create a PrettyTable object\n", + "table = PrettyTable()\n", + "\n", + "# Add column names to the table\n", + "table.field_names = column_names\n", + "\n", + "# Set max width for each column to truncate data\n", + "table.max_width = 20\n", + "\n", + "# Add rows to the table\n", + "for row in queryresults:\n", + " # Truncate each value to 20 characters\n", + " truncated_row = [str(value)[:20] for value in row]\n", + " table.add_row(truncated_row)\n", + "\n", + "# Print the table\n", + "print(table)\n", + "\n", + "# Commit the changes\n", + "conn.commit()\n", + "# Close the connection\n", + "conn.close()\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "b7b0fda6-3322-4bbc-8d08-d0b567da79ec", + "language": "python" + }, + "source": [ + "### **Performing Vector Similarity Search in Azure SQL DB using VECTOR\\_DISTANCE built in function**\n", + "\n", + "Let's now query our ResumeDocs table to get the top similar candidates given the User search query.\n", + "\n", + "What we are doing: Given any user search query, we can obtain the vector representation of that text. We then use this vector to calculate the cosine distance against all the resume embeddings stored in the database. By selecting only the closest matches, we can identify the resumes most relevant to the user’s query. This helps in finding the most suitable candidates based on their resumes.\n", + "\n", + "The most common distance is the cosine similarity, which can be calculated quite easily in SQL with the help of the new distance functions.\n", + "\n", + "```\n", + "VECTOR_DISTANCE('distance metric', V1, V2)\n", + "\n", + "```\n", + "\n", + "We can use **cosine**, **euclidean**, and **dot** as the distance metric today.\n", + "\n", + "We will define the function `vector_search_sql`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "1b4f0ca2-2401-4f90-a44d-75dce03500cc", + "language": "python" + }, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "from dotenv import load_dotenv\n", + "\n", + "def vector_search_sql(query, num_results=5):\n", + " # Load environment variables from .env file\n", + " load_dotenv()\n", + "\n", + " # Use the get_mssql_connection function to get the connection\n", + " conn = get_mssql_connection()\n", + "\n", + " # Create a cursor object\n", + " cursor = conn.cursor()\n", + "\n", + " # Generate the query embedding for the user's search query\n", + " user_query_embedding = get_embedding(query)\n", + "\n", + " # SQL similarity search via VECTOR_DISTANCE. The query vector goes over the\n", + " # wire as JSON text, then CAST twice: NVARCHAR(MAX) first (mssql-python\n", + " # binds Python strings as ntext), then VECTOR(1536).\n", + " sql_similarity_search = \"\"\"\n", + " SELECT TOP(?) filename, chunkid, chunk,\n", + " 1 - VECTOR_DISTANCE('cosine', CAST(CAST(? AS NVARCHAR(MAX)) AS VECTOR(1536)), embedding) AS similarity_score,\n", + " VECTOR_DISTANCE('cosine', CAST(CAST(? AS NVARCHAR(MAX)) AS VECTOR(1536)), embedding) AS distance_score\n", + " FROM dbo.resumedocs\n", + " ORDER BY distance_score\n", + " \"\"\"\n", + "\n", + " cursor.execute(sql_similarity_search,\n", + " num_results,\n", + " json.dumps(user_query_embedding),\n", + " json.dumps(user_query_embedding))\n", + " results = cursor.fetchall()\n", + "\n", + " # Close the database connection\n", + " conn.close()\n", + " return results\n", + "\n", + "# example usage\n", + "vector_search_sql(\"database administrator\", num_results=3)\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "azdata_cell_guid": "1e194f3c-6a7a-4f16-95ec-05f60a3770a4", + "language": "python" + }, + "source": [ + "# **Part 4 : Use embeddings retrieved from a Azure SQL vector database to augment LLM generation**\n", + "\n", + "Lets create a helper function to feed prompts into the [Completions model](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#gpt-4) & create interactive loop where you can pose questions to the model and receive information grounded in your data.\n", + "\n", + "The function `generate_completion` is defined to help ground the gpt-4o model with prompts and system instructions.  \n", + "Note that we are passing the results of the `vector_search_sql` we defined earlier to the model and we define the system prompt . \n", + "We are using gpt-4o model here. \n", + "\n", + "You can get more information on using Azure Open AI GPT chat models [here](https://learn.microsoft.com/azure/ai-services/openai/chatgpt-quickstart?tabs=command-line%2Cpython-new&pivots=programming-language-python)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "2865a0d0-2ee3-4d0a-8ee8-89075ab541bc", + "language": "python" + }, + "outputs": [], + "source": [ + "import os\n", + "from dotenv import load_dotenv\n", + "from openai import AzureOpenAI\n", + "\n", + "# Load environment variables from a .env file\n", + "load_dotenv()\n", + "\n", + "# Use environment variables for the API key and endpoint\n", + "api_key = os.getenv(\"AZOPENAI_API_KEY\")\n", + "azure_endpoint = os.getenv(\"AZOPENAI_ENDPOINT\")\n", + "chat_model = os.getenv(\"AZOPENAI_CHAT_MODEL_DEPLOYMENT_NAME\")\n", + "\n", + "# Create a chat completion request\n", + "client = AzureOpenAI(\n", + " api_key=api_key,\n", + " api_version=\"2023-05-15\",\n", + " azure_endpoint=azure_endpoint\n", + ")\n", + "\n", + "def generate_completion(search_results, user_input):\n", + " system_prompt = '''\n", + "You are an intelligent & funny assistant who will exclusively answer based on the data provided in the `search_results`:\n", + "- Use the information from `search_results` to generate your top 3 responses. If the data is not a perfect match for the user's query, use your best judgment to provide helpful suggestions and include the following format:\n", + " File: {filename}\n", + " Chunk ID: {chunkid}\n", + " Similarity Score: {similarity_score}\n", + " Add a small snippet from the Relevant Text: {chunktext}\n", + " Do not use the entire chunk\n", + "- Avoid any other external data sources.\n", + "- Add a summary about why the candidate maybe a goodfit even if exact skills and the role being hired for are not matching , at the end of the recommendations. Ensure you call out which skills match the description and which ones are missing. If the candidate doesnt have prior experience for the hiring role which we may need to pay extra attention to during the interview process.\n", + "- Add a Microsoft related interesting fact about the technology that was searched \n", + "'''\n", + "\n", + " messages = [{\"role\": \"system\", \"content\": system_prompt}]\n", + " \n", + " # Create an empty list to store the results\n", + " result_list = []\n", + "\n", + " # Iterate through the search results and append relevant information to the list\n", + " for result in search_results:\n", + " filename = result # Assuming filename is the first column\n", + " chunkid = result\n", + " chunktext = result\n", + " similarity_score = result # Assuming similarity_score is the third column\n", + " \n", + " # Append the relevant information as a dictionary to the result_list\n", + " result_list.append({\n", + " \"filename\": filename,\n", + " \"chunkid\": chunkid,\n", + " \"chunktext\": chunktext,\n", + " \"similarity_score\": similarity_score\n", + " })\n", + "\n", + " # Print the result list\n", + " #print(result_list)\n", + " \n", + " messages.append({\"role\": \"system\", \"content\": f\"{result_list}\"})\n", + " messages.append({\"role\": \"user\", \"content\": user_input})\n", + " response = client.chat.completions.create(model=chat_model, messages=messages, temperature=0) \n", + "\n", + " return response.dict()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "azdata_cell_guid": "d69ab285-0cc9-4596-95f2-688d0c324f6b", + "language": "python", + "tags": [] + }, + "outputs": [], + "source": [ + "# Create a loop of user input and model output to perform Q&A on the PDF's that are now chunked and stored in the SQL DB with embeddings\n", + "#\n", + "# PLEASE NOTE: An input box will be displayed for the user to enter a question/query at the top of the scree.\n", + "# The model will then provide a response based on the data stored in the SQL DB.\n", + "# Type 'end' to end the session.\n", + "#\n", + "print(\"*** What Role are you hiring for? And What skills are you looking for? Ask me & I can help you find a candidate :) Type 'end' to end the session.\\n\")\n", + "\n", + "while True:\n", + " user_input = input(\"User prompt: \")\n", + " if user_input.lower() == \"end\":\n", + " break\n", + "\n", + " # Print the user's question\n", + " print(f\"\\nUser asked: {user_input}\")\n", + "\n", + " \n", + " # Assuming vector_search_sql and generate_completion are defined functions that work correctly\n", + " search_results = vector_search_sql(user_input)\n", + " completions_results = generate_completion(search_results, user_input)\n", + "\n", + " # Print the model's response\n", + " print(\"\\nAI's response:\")\n", + " print(completions_results['choices'][0]['message']['content'])\n", + "\n", + "# The loop will continue until the user types 'end'\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } From fe5bb14628fe4a401771f5f4be1b0e3ff2e571b0 Mon Sep 17 00:00:00 2001 From: Anna Hoffman Date: Fri, 24 Jul 2026 12:06:04 -0400 Subject: [PATCH 3/5] Address second round of Copilot PR review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - RAG-with-resumes.ipynb: generate_completion() was treating each row of search_results as a single scalar (filename = result, chunkid = result, etc.), so the LLM grounding payload had every field set to the whole row tuple. Fixed to unpack the (filename, chunkid, chunk, similarity_score, distance_score) shape that vector_search_sql returns. - FineFoodReviews/_load-reviews.py: drop the misleading 'triggers AI_GENERATE_EMBEDDINGS server-side' line from the docstring — this script only loads rows; embedding happens in 001-load-and-embed.sql or _embed-reviews.py. --- DiskANN/FineFoodReviews/_load-reviews.py | 9 ++++++--- RAG-with-Documents/RAG-with-resumes.ipynb | 13 ++++--------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/DiskANN/FineFoodReviews/_load-reviews.py b/DiskANN/FineFoodReviews/_load-reviews.py index d67a545..6e7e61b 100644 --- a/DiskANN/FineFoodReviews/_load-reviews.py +++ b/DiskANN/FineFoodReviews/_load-reviews.py @@ -1,6 +1,9 @@ -"""Load first 500 rows of Datasets/Reviews.csv into dbo.reviews, then -trigger AI_GENERATE_EMBEDDINGS server-side. Uses mssql-python (the native -Microsoft first-party Python driver) with Entra Default auth — no token juggling. +"""Load first 500 rows of Datasets/Reviews.csv into dbo.reviews. +Uses mssql-python (the native Microsoft first-party Python driver) with +Entra Default auth — no token juggling. + +Embeddings are generated in a separate step: either 001-load-and-embed.sql +(pure T-SQL) or _embed-reviews.py (batched from Python). Configure via environment variables (or edit the defaults below): MSSQL_SERVER e.g. myserver.database.windows.net diff --git a/RAG-with-Documents/RAG-with-resumes.ipynb b/RAG-with-Documents/RAG-with-resumes.ipynb index 8d2ef70..bbbfc02 100644 --- a/RAG-with-Documents/RAG-with-resumes.ipynb +++ b/RAG-with-Documents/RAG-with-resumes.ipynb @@ -753,19 +753,14 @@ " # Create an empty list to store the results\n", " result_list = []\n", "\n", - " # Iterate through the search results and append relevant information to the list\n", - " for result in search_results:\n", - " filename = result # Assuming filename is the first column\n", - " chunkid = result\n", - " chunktext = result\n", - " similarity_score = result # Assuming similarity_score is the third column\n", - " \n", - " # Append the relevant information as a dictionary to the result_list\n", + " # Iterate through the search results and append relevant information to the list.\n", + " # vector_search_sql returns rows shaped (filename, chunkid, chunk, similarity_score, distance_score)\n", + " for filename, chunkid, chunktext, similarity_score, _distance in search_results:\n", " result_list.append({\n", " \"filename\": filename,\n", " \"chunkid\": chunkid,\n", " \"chunktext\": chunktext,\n", - " \"similarity_score\": similarity_score\n", + " \"similarity_score\": similarity_score,\n", " })\n", "\n", " # Print the result list\n", From 7faec68e3a81ae2feb34fea8ffa057c8173d8861 Mon Sep 17 00:00:00 2001 From: "Anna Hoffman (Thomas)" Date: Fri, 24 Jul 2026 13:20:33 -0400 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- DiskANN/diskann-quickstart-azure-sql-improvements.sql | 2 +- DiskANN/diskann-quickstart-azure-sql.sql | 4 ++-- DiskANN/diskann-quickstart-sql-server-2025.sql | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DiskANN/diskann-quickstart-azure-sql-improvements.sql b/DiskANN/diskann-quickstart-azure-sql-improvements.sql index d44313f..5b006b2 100644 --- a/DiskANN/diskann-quickstart-azure-sql-improvements.sql +++ b/DiskANN/diskann-quickstart-azure-sql-improvements.sql @@ -338,7 +338,7 @@ SELECT last_background_task_processed_inserts, last_background_task_processed_deletes FROM sys.dm_db_vector_indexes -WHERE OBJECT_NAME(object_id) = 'Articles'; +WHERE object_id = OBJECT_ID(N'dbo.Articles'); GO -- What you observed: diff --git a/DiskANN/diskann-quickstart-azure-sql.sql b/DiskANN/diskann-quickstart-azure-sql.sql index fc70d2a..f6e9f5d 100644 --- a/DiskANN/diskann-quickstart-azure-sql.sql +++ b/DiskANN/diskann-quickstart-azure-sql.sql @@ -11,7 +11,7 @@ CREATE TABLE dbo.Articles -- Step 2: Insert sample data -- 10 named rows for storytelling + 90 generated rows. -- DiskANN requires at least 100 non-null vectors to build the index. -INSERT INTO Articles (id, title, content, embedding) +INSERT INTO dbo.Articles (id, title, content, embedding) VALUES (1, 'Intro to AI', 'This article introduces AI concepts.', '[0.1, 0.2, 0.3, 0.4, 0.5]'), (2, 'Deep Learning', 'Deep learning is a subset of ML.', '[0.2, 0.1, 0.4, 0.3, 0.6]'), @@ -106,7 +106,7 @@ SELECT last_background_task_processed_inserts, last_background_task_processed_deletes FROM sys.dm_db_vector_indexes -WHERE OBJECT_NAME(object_id) = 'Articles'; +WHERE object_id = OBJECT_ID(N'dbo.Articles'); GO -- Step 9: Clean up diff --git a/DiskANN/diskann-quickstart-sql-server-2025.sql b/DiskANN/diskann-quickstart-sql-server-2025.sql index 3bd13d4..89117da 100644 --- a/DiskANN/diskann-quickstart-sql-server-2025.sql +++ b/DiskANN/diskann-quickstart-sql-server-2025.sql @@ -24,7 +24,7 @@ CREATE TABLE dbo.Articles -- Step 2: Insert sample data -- 10 named rows for storytelling + 90 generated rows. -- DiskANN requires at least 100 non-null vectors to build the index. -INSERT INTO Articles (id, title, content, embedding) +INSERT INTO dbo.Articles (id, title, content, embedding) VALUES (1, 'Intro to AI', 'This article introduces AI concepts.', '[0.1, 0.2, 0.3, 0.4, 0.5]'), (2, 'Deep Learning', 'Deep learning is a subset of ML.', '[0.2, 0.1, 0.4, 0.3, 0.6]'), From 1c197c8f6c162778b568566fdc3989757786b69c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:33:29 +0000 Subject: [PATCH 5/5] Fix hard-coded path and file_name loop in RAG notebook - Replace hard-coded Windows path with os.getenv("RESUME_FOLDER_PATH", ".") - Remove separate loop that computed file_name only for the last file - Compute file_name = os.path.basename(pdf_file) inside the enumerate loop - Use pdf_file directly as pdf_path (already a full path from get_pdf_files)" --- RAG-with-Documents/RAG-with-resumes.ipynb | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/RAG-with-Documents/RAG-with-resumes.ipynb b/RAG-with-Documents/RAG-with-resumes.ipynb index bbbfc02..704e36f 100644 --- a/RAG-with-Documents/RAG-with-resumes.ipynb +++ b/RAG-with-Documents/RAG-with-resumes.ipynb @@ -169,7 +169,7 @@ "import tiktoken\n", "\n", "# Path to the directory containing PDF files\n", - "folder_path = os.path.join(os.getcwd(),'C:\\\\vectortest\\\\resumedata\\\\data\\\\data\\\\INFORMATION-TECHNOLOGY')\n", + "folder_path = os.getenv(\"RESUME_FOLDER_PATH\", \".\")\n", "\n", "def get_pdf_files(folder_path):\n", " for path, subdirs, files in os.walk(folder_path):\n", @@ -212,16 +212,14 @@ "num_files = len(pdf_files)\n", "print(f\"Number of PDF files in the directory: {num_files}\")\n", "\n", - "#Extract the file name\n", - "for pdf_file in pdf_files:\n", - " file_name = os.path.basename(pdf_file)\n", "\n", "# Create a DataFrame to store the chunks\n", "data = []\n", "\n", "for file_id, pdf_file in enumerate(pdf_files):\n", + " file_name = os.path.basename(pdf_file)\n", " print(f\"Processing file {file_id + 1}/{num_files}: {file_name}\")\n", - " pdf_path = os.path.join(folder_path, pdf_file)\n", + " pdf_path = pdf_file\n", " text = extract_text_from_pdf(pdf_path)\n", " cleaned_text = clean_text(text)\n", " chunks = split_text_into_token_chunks(cleaned_text)\n",