-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathstack.env
More file actions
159 lines (143 loc) · 7.16 KB
/
Copy pathstack.env
File metadata and controls
159 lines (143 loc) · 7.16 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
# ============================================================================
# Portainer Stack Environment Variables
# ============================================================================
# This file is used by Portainer when deploying via Git repository.
# Variables here override docker-compose.prod.yml defaults.
# ============================================================================
# ----------------------------------------------------------------------------
# PostgreSQL Configuration
# ----------------------------------------------------------------------------
POSTGRES_USER=user
POSTGRES_PASSWORD=password
POSTGRES_DB=fileintel
POSTGRES_MAX_CONNECTIONS=600
POSTGRES_SHARED_BUFFERS=3GB
POSTGRES_EFFECTIVE_CACHE_SIZE=8GB
POSTGRES_WORK_MEM=32MB
POSTGRES_MEMORY_LIMIT=10G
# ----------------------------------------------------------------------------
# Redis Configuration
# ----------------------------------------------------------------------------
REDIS_MAXMEMORY=3gb
REDIS_MEMORY_LIMIT=4G
# ----------------------------------------------------------------------------
# Storage Configuration
# ----------------------------------------------------------------------------
STORAGE_POOL_SIZE=20
STORAGE_MAX_OVERFLOW=30
STORAGE_POOL_TIMEOUT=300
# ----------------------------------------------------------------------------
# API Configuration
# ----------------------------------------------------------------------------
API_PORT=8001
API_KEY=key
# ----------------------------------------------------------------------------
# LLM Provider Configuration
# ----------------------------------------------------------------------------
OPENAI_API_KEY=key
# HTTP timeout for LLM requests
# IMPORTANT: With high concurrency (200 parallel requests), the LLM server queues requests
# Each request waits in queue + processes, so total time can be much longer than inference time
# Formula: HTTP_TIMEOUT = (avg_inference_time × max_queue_depth) + buffer
# Example: 30s inference × 100 queue depth = 3000s + buffer = 3600s (1 hour)
LLM_HTTP_TIMEOUT=3600 # 1 hour (handles high queue depths with 200 concurrent requests)
LLM_MAX_RETRIES=20
LLM_RETRY_BACKOFF_MIN=2
LLM_RETRY_BACKOFF_MAX=300
# ----------------------------------------------------------------------------
# Logging Configuration
# ----------------------------------------------------------------------------
LOG_LEVEL=INFO
TRANSFORMERS_NO_ADVISORY_WARNINGS=1
# ----------------------------------------------------------------------------
# Host Paths (TrueNAS mounts)
# ----------------------------------------------------------------------------
HOST_PROMPTS_PATH=/mnt/HDDs/docker/apps/fileintel/prompts
HOST_LOGS_PATH=/mnt/HDDs/docker/apps/fileintel/logs
HOST_UPLOADS_PATH=/mnt/HDDs/docker/apps/fileintel/uploads
HOST_INPUT_PATH=/mnt/HDDs/docker/apps/fileintel/input
HOST_OUTPUT_PATH=/mnt/HDDs/docker/apps/fileintel/output
HOST_GRAPHRAG_PATH=/mnt/HDDs/docker/apps/fileintel/graphrag_index
HOST_MINERU_OUTPUTS_PATH=/mnt/HDDs/docker/apps/fileintel/mineru
# ----------------------------------------------------------------------------
# Celery Worker Configuration - Prefork
# ----------------------------------------------------------------------------
CELERY_WORKER_POOL=prefork
CELERY_WORKER_CONCURRENCY=6
CELERY_WORKER_MEMORY_LIMIT=20G
CELERY_WORKER_MEMORY_RESERVATION=8G
CELERY_MEMORY_OVERHEAD_GB=2.0
# ----------------------------------------------------------------------------
# Celery Worker Configuration - Gevent (GraphRAG)
# ----------------------------------------------------------------------------
CELERY_GRAPHRAG_CONCURRENCY=200
CELERY_GRAPHRAG_MEMORY_LIMIT=24G
CELERY_GRAPHRAG_MEMORY_RESERVATION=4G
# ----------------------------------------------------------------------------
# GraphRAG Configuration - CRITICAL
# ----------------------------------------------------------------------------
# Leiden clustering configuration - THESE ARE THE KEY VARIABLES
# Target base community size: 15-25 entities (not used directly by pyramid algorithm, kept for reference)
GRAPHRAG_MAX_CLUSTER_SIZE=25
# Resolution: Controls base granularity (higher = more fine-grained base communities)
# 1.0 is recommended for ~75K entities to create ~3000-5000 base communities
GRAPHRAG_LEIDEN_RESOLUTION=1.0
# Adaptive scaling: Automatically adjusts clustering for different collection sizes
# When enabled, overrides BASE_RESOLUTION_MULTIPLIER based on graph size
GRAPHRAG_USE_ADAPTIVE_SCALING=true
# Target average entities per base community (recommended: 15-25)
GRAPHRAG_ADAPTIVE_TARGET_BASE_SIZE=20
# Base resolution multiplier: Only used if adaptive scaling is disabled
# Higher values = more base communities (220.0 for ~5500 base communities with 85K entities)
# Formula: base_resolution = LEIDEN_RESOLUTION × BASE_RESOLUTION_MULTIPLIER
GRAPHRAG_BASE_RESOLUTION_MULTIPLIER=220.0
# Consolidation scaling factor: Controls pyramid steepness (number of levels)
# Lower values = steeper consolidation (fewer levels), higher values = gentler (more levels)
# 0.3 = steep/optimal (6 levels), 0.35 = moderate (6-7 levels), 0.4 = gentle (7-8 levels)
GRAPHRAG_CONSOLIDATION_SCALING_FACTOR=0.30
# Enable pyramid hierarchy algorithm (replaces hierarchical_leiden)
GRAPHRAG_USE_PYRAMID_HIERARCHY=true
# GraphRAG async processing (matches gevent concurrency)
GRAPHRAG_ASYNC_ENABLED=true
GRAPHRAG_ASYNC_BATCH_SIZE=8
GRAPHRAG_ASYNC_MAX_CONCURRENT=200
# GraphRAG retry settings
GRAPHRAG_MAX_RETRIES=200
GRAPHRAG_MAX_RETRY_WAIT=300
# ----------------------------------------------------------------------------
# GraphRAG Timeout Configuration (SIMPLIFIED)
# ----------------------------------------------------------------------------
# Two-layer timeout strategy for GraphRAG indexing:
#
# Layer 1: HTTP Timeout (handles queue wait + inference)
# └─ LLM_HTTP_TIMEOUT=3600 (1 hour per HTTP request)
# └─ Detects: network hangs, server unresponsiveness
# └─ Accounts for: queue depth + processing time with 200 concurrent requests
#
# Layer 2: Workflow Inactivity Timeout (PRIMARY DEFENSE)
# └─ GRAPHRAG__WORKFLOW_TIMEOUT=10800 (3 hours of no progress)
# └─ Heartbeat-based: resets on every LLM call and progress update
# └─ Detects: stuck workflows (no heartbeats for 3 hours)
# └─ Allows: long-running workflows that make steady progress
#
# Why this works:
# - With 200 concurrent requests, each request waits in LLM server queue
# - HTTP timeout (1 hour) allows for queue wait + processing
# - Workflow timeout (3 hours) only triggers if NO requests complete
# - Heartbeats signal every LLM completion, resetting workflow timeout
#
# REMOVED (redundant):
# - GRAPHRAG_REQUEST_TIMEOUT (redundant with LLM_HTTP_TIMEOUT)
# - GRAPHRAG_ASYNC_BATCH_TIMEOUT (redundant with workflow timeout)
# ----------------------------------------------------------------------------
GRAPHRAG__WORKFLOW_TIMEOUT=10800
# ----------------------------------------------------------------------------
# Vector RAG Configuration
# ----------------------------------------------------------------------------
RAG_ASYNC_ENABLED=true
RAG_ASYNC_BATCH_SIZE=8
RAG_ASYNC_MAX_CONCURRENT=50
RAG_ASYNC_BATCH_TIMEOUT=9000
RAG_EMBEDDING_BATCH_SIZE=50
RAG_EMBEDDING_FALLBACK_SINGLE=true
RAG_EMBEDDING_RETRY_INDIVIDUAL=true