Skip to content

Commit a86a453

Browse files
committed
Modernize wikimon: Python 3 / asyncio / EventStreams / WSS
Replace Python 2.7 / Twisted / IRC / autobahn stack with: - Python 3.10+ / asyncio / aiohttp / websockets - Wikimedia EventStreams (SSE) instead of IRC - maxminddb instead of python-geoip - Direct MediaWiki API call instead of wapiti New files: - wikimon/monitor.py: asyncio-based EventStreams consumer + WS broadcaster - wikimon/__main__.py: python -m wikimon.monitor entry point - wikimon/test_monitor.py: 24 tests for transform_event + GeoIPManager - conf/supervisord.conf: all 43 language processes - Dockerfile + docker-compose.yml: local dev/testing Modified: - wikimon/parsers.py: Python 3 imports, new parse_comment() function - wikimon/test_parsers.py: 32 proper pytest tests (was print-only) - conf/wikimon.nginx.conf: all 43 location blocks, fixed Host header - requirements.txt: aiohttp, websockets, maxminddb, requests, pytest - setup.py: Python 3.10+, new deps, version 0.7.0 Deleted: - monitor_websocket_wikidata.py: folded into monitor.py (--project wikidata) Output format preserved exactly for frontend compatibility. Resolves listen-to-wikipedia #28, #38, #12, #48, #31.
1 parent 15e5e2c commit a86a453

13 files changed

Lines changed: 1669 additions & 502 deletions

Dockerfile

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
FROM python:3.10-slim
2+
3+
WORKDIR /app
4+
5+
COPY requirements.txt .
6+
RUN pip install --no-cache-dir -r requirements.txt
7+
8+
COPY . .
9+
10+
EXPOSE 9000
11+
12+
ENTRYPOINT ["python", "-m", "wikimon.monitor"]

conf/supervisord.conf

Lines changed: 386 additions & 0 deletions
Large diffs are not rendered by default.

conf/wikimon.nginx.conf

Lines changed: 69 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,73 @@
11
server {
2-
server_name wikimon.hatnote.com;
3-
root /home/hatnote/wikimon/static/;
4-
access_log /home/hatnote/wikimon/logs/access.log combined buffer=128k flush=10s;
5-
error_log /home/hatnote/wikimon/logs/error.log;
6-
7-
expires 1d;
8-
9-
proxy_http_version 1.1;
10-
proxy_set_header Upgrade $http_upgrade;
11-
proxy_set_header Connection "Upgrade";
12-
proxy_set_header Host "${host}:${proxy_port}";
13-
14-
location / {
15-
proxy_pass http://127.0.0.1:9000;
16-
}
17-
18-
location /en/ {
19-
proxy_pass http://127.0.0.1:9000;
20-
}
21-
22-
location /de/ {
23-
proxy_pass http://127.0.0.1:9010;
24-
}
25-
26-
location /ru/ {
27-
proxy_pass http://127.0.0.1:9020;
28-
}
29-
30-
location /ja/ {
31-
proxy_pass http://127.0.0.1:9030;
32-
}
33-
34-
location /test/ {
35-
proxy_pass http://127.0.0.1:9999;
36-
proxy_buffering off;
37-
}
38-
39-
listen 443 ssl; # managed by Certbot
40-
ssl_certificate /etc/letsencrypt/live/hatnote.com/fullchain.pem; # managed by Certbot
41-
ssl_certificate_key /etc/letsencrypt/live/hatnote.com/privkey.pem; # managed by Certbot
42-
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
43-
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
44-
45-
46-
47-
48-
49-
2+
server_name wikimon.hatnote.com;
3+
listen 443 ssl;
4+
5+
ssl_certificate /etc/letsencrypt/live/hatnote.com/fullchain.pem;
6+
ssl_certificate_key /etc/letsencrypt/live/hatnote.com/privkey.pem;
7+
include /etc/letsencrypt/options-ssl-nginx.conf;
8+
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
9+
10+
access_log /home/hatnote/wikimon/logs/access.log combined buffer=128k flush=10s;
11+
error_log /home/hatnote/wikimon/logs/error.log;
12+
13+
proxy_http_version 1.1;
14+
proxy_set_header Upgrade $http_upgrade;
15+
proxy_set_header Connection "Upgrade";
16+
proxy_set_header Host $host;
17+
proxy_set_header X-Real-IP $remote_addr;
18+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
19+
proxy_set_header X-Forwarded-Proto $scheme;
20+
proxy_read_timeout 86400s;
21+
proxy_send_timeout 86400s;
22+
23+
location / { proxy_pass http://127.0.0.1:9000; }
24+
location /en/ { proxy_pass http://127.0.0.1:9000; }
25+
location /de/ { proxy_pass http://127.0.0.1:9010; }
26+
location /ru/ { proxy_pass http://127.0.0.1:9020; }
27+
location /ja/ { proxy_pass http://127.0.0.1:9030; }
28+
location /es/ { proxy_pass http://127.0.0.1:9040; }
29+
location /fr/ { proxy_pass http://127.0.0.1:9050; }
30+
location /nl/ { proxy_pass http://127.0.0.1:9060; }
31+
location /it/ { proxy_pass http://127.0.0.1:9070; }
32+
location /sv/ { proxy_pass http://127.0.0.1:9080; }
33+
location /ar/ { proxy_pass http://127.0.0.1:9090; }
34+
location /id/ { proxy_pass http://127.0.0.1:9100; }
35+
location /ta/ { proxy_pass http://127.0.0.1:9110; }
36+
location /pa/ { proxy_pass http://127.0.0.1:9120; }
37+
location /mr/ { proxy_pass http://127.0.0.1:9130; }
38+
location /hi/ { proxy_pass http://127.0.0.1:9140; }
39+
location /as/ { proxy_pass http://127.0.0.1:9150; }
40+
location /bn/ { proxy_pass http://127.0.0.1:9160; }
41+
location /te/ { proxy_pass http://127.0.0.1:9165; }
42+
location /kn/ { proxy_pass http://127.0.0.1:9170; }
43+
location /or/ { proxy_pass http://127.0.0.1:9180; }
44+
location /sa/ { proxy_pass http://127.0.0.1:9190; }
45+
location /gu/ { proxy_pass http://127.0.0.1:9200; }
46+
location /fa/ { proxy_pass http://127.0.0.1:9210; }
47+
location /wikidata/ { proxy_pass http://127.0.0.1:9220; }
48+
location /he/ { proxy_pass http://127.0.0.1:9230; }
49+
location /zh/ { proxy_pass http://127.0.0.1:9240; }
50+
location /ml/ { proxy_pass http://127.0.0.1:9250; }
51+
location /pl/ { proxy_pass http://127.0.0.1:9260; }
52+
location /mk/ { proxy_pass http://127.0.0.1:9270; }
53+
location /be/ { proxy_pass http://127.0.0.1:9280; }
54+
location /sr/ { proxy_pass http://127.0.0.1:9290; }
55+
location /bg/ { proxy_pass http://127.0.0.1:9300; }
56+
location /uk/ { proxy_pass http://127.0.0.1:9310; }
57+
location /hu/ { proxy_pass http://127.0.0.1:9320; }
58+
location /fi/ { proxy_pass http://127.0.0.1:9330; }
59+
location /no/ { proxy_pass http://127.0.0.1:9340; }
60+
location /el/ { proxy_pass http://127.0.0.1:9350; }
61+
location /eo/ { proxy_pass http://127.0.0.1:9360; }
62+
location /pt/ { proxy_pass http://127.0.0.1:9370; }
63+
location /et/ { proxy_pass http://127.0.0.1:9380; }
64+
location /ur/ { proxy_pass http://127.0.0.1:9390; }
65+
location /ro/ { proxy_pass http://127.0.0.1:9400; }
66+
location /hy/ { proxy_pass http://127.0.0.1:9410; }
5067
}
5168

5269
server {
53-
if ($host = wikimon.hatnote.com) {
54-
return 301 https://$host$request_uri;
55-
} # managed by Certbot
56-
57-
58-
server_name wikimon.hatnote.com;
59-
listen 80;
60-
return 404; # managed by Certbot
61-
62-
63-
}
70+
server_name wikimon.hatnote.com;
71+
listen 80;
72+
return 301 https://$host$request_uri;
73+
}

docker-compose.yml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
services:
2+
wikimon-en:
3+
build: .
4+
ports:
5+
- "9000:9000"
6+
command: ["--lang", "en", "--port", "9000", "--loglevel", "INFO"]
7+
restart: unless-stopped
8+
9+
wikimon-de:
10+
build: .
11+
ports:
12+
- "9010:9010"
13+
command: ["--lang", "de", "--port", "9010", "--loglevel", "INFO"]
14+
restart: unless-stopped
15+
profiles: ["full"]
16+
17+
wikimon-wikidata:
18+
build: .
19+
ports:
20+
- "9220:9220"
21+
command: ["--lang", "wikidata", "--project", "wikidata", "--port", "9220", "--loglevel", "INFO"]
22+
restart: unless-stopped
23+
profiles: ["full"]

0 commit comments

Comments
 (0)