Merge branch 'master' of github.com:hatnote/wikimon

Author: mahmoud

README.md

@@ -43,32 +43,52 @@ optional arguments:
 Here are a couple example messages, as broadcast over WebSocket:
 
 ```json
-
-{"is_minor": false,
- "page_title": "Template:Citation needed/testcases",
- "url": "http://en.wikipedia.org/w/index.php?diff=553804313&oldid=479472901",
- "is_unpatrolled": false,
- "is_bot": false,
- "is_new": false,
- "summary": null,
- "flags": null,
- "user": "98.172.160.184",
- "is_anon": true,
- "ns": "Template",
- "change_size": "+42"}
-
-{"is_minor": true,
- "page_title": "Generalized anxiety disorder",
- "url": "http://en.wikipedia.org/w/index.php?diff=553804315&oldid=553370901",
- "is_unpatrolled": false,
- "is_bot": false,
- "is_new": false,
- "summary": "minor editing in sentences.",
- "flags": "M",
- "user": "BriannaMaxim",
- "is_anon": false,
- "ns": "Main",
- "change_size": "+1"}
+{
+  "action": "edit",
+  "change_size": 19,
+  "flags": "M",
+  "hashtags": [],
+  "is_anon": false,
+  "is_bot": false,
+  "is_minor": true,
+  "is_new": false,
+  "is_unpatrolled": false,
+  "mentions": [],
+  "ns": "User talk",
+  "page_title": "User talk:Manxruler",
+  "parent_rev_id": "775894803",
+  "rev_id": "775894650",
+  "summary": "/* The battle of Kristiansand (1940) */",
+  "url": "https://en.wikipedia.org/w/index.php?diff=775894803&oldid=775894650",
+  "user": "Carsten R D"
+}
+
+{
+  "action": "edit",
+  "change_size": -12,
+  "flags": null,
+  "geo_ip": {
+    "city": "Salisbury",
+    "country_name": "United States",
+    "latitude": 38.3761,
+    "longitude": -75.6086,
+    "region_name": "Maryland"
+  },
+  "hashtags": [],
+  "is_anon": true,
+  "is_bot": false,
+  "is_minor": false,
+  "is_new": false,
+  "is_unpatrolled": false,
+  "mentions": [],
+  "ns": "Main",
+  "page_title": "Evanescence (Evanescence album)",
+  "parent_rev_id": "775894800",
+  "rev_id": "774995266",
+  "summary": "/* Credits and personnel */ \"Personnel\" is sufficient",
+  "url": "https://en.wikipedia.org/w/index.php?diff=775894800&oldid=774995266",
+  "user": "71.200.123.192"
+}
 ```
 
 As you can see, the set of keys sent is always the same. Note that the
@@ -77,16 +97,7 @@ As you can see, the set of keys sent is always the same. Note that the
 
 ## Geolocation
 
-Geolocation is currently done through a local FreeGeoIP
-instance. FreeGeoIP requires Go and several Go libraries. It also
-requires memcached to be running on port 11211.
-
-The command used to run FreeGeoIP at the moment:
-
-```
-GOPATH=/home/hatnote/gopkg/ GOROOT=/home/hatnote/go nohup /home/hatnote/go/bin/go run freegeoip.go &
-```
-
+Geolocation is done in process, using maxmind's free dataset. See the GeoDB directory for more info.
 
 ## See also
 

requirements.txt

@@ -1,7 +1,8 @@
--e git+https://github.com/mahmoud/wapiti.git#egg=wapiti
-
-Twisted>=13.1.0
 autobahn==0.5.14
+py==1.4.26
 pytest==2.6.1
-python-geoip
-python-geoip-geolite2
+python-geoip==1.2
+python-geoip-geolite2==2014.207
+Twisted==14.0.2
+-e git+https://github.com/mahmoud/wapiti.git@5dc462b1e0d2290c1841980c94309efe4c27b75a#egg=wapiti-master
+zope.interface==4.1.2

setup.py

@@ -7,7 +7,7 @@
     A WebSocket-oriented monitor for streaming live changes to
     Wikipedia. (also, wikimon, wikital monsters)
 
-    :copyright: (c) 2013 by Mahmoud Hashemi and Stephen LaPorte
+    :copyright: (c) 2013-2015 by Mahmoud Hashemi and Stephen LaPorte
     :license: GPLv3, see LICENSE for more details.
 
 """
@@ -17,7 +17,7 @@
 
 
 __author__ = 'Mahmoud Hashemi and Stephen LaPorte'
-__version__ = '0.5.2'
+__version__ = '0.6.3dev'
 __contact__ = 'mahmoudrhashemi@gmail.com'
 __url__ = 'https://github.com/hatnote/wikimon'
 __license__ = 'GPLv3'
@@ -44,7 +44,11 @@
       packages=['wikimon'],
       include_package_data=True,
       zip_safe=False,
-      install_requires=['wapiti', 'Twisted==13.0.0', 'autobahn==0.5.14'],
+      install_requires=['wapiti',
+                        'Twisted==13.0.0',
+                        'autobahn==0.5.14',
+                        'python-geoip==1.2',
+                        'python-geoip-geolite2==2014.207'],
       license=__license__,
       platforms='any',
       classifiers=[

wikimon/parsers.py

@@ -16,6 +16,9 @@
 HASHTAG_RE = re.compile("(?:^|\s)[＃#]{1}(\w+)", re.UNICODE)
 MENTION_RE = re.compile("(?:^|\s)[＠ @]{1}([^\s#<>[\]|{}]+)", re.UNICODE)
 
+_SECTION_TITLE_RE = re.compile("\/\*\s*(?P<section_title>.+)\s*\*\/"
+                               "(?P<real_summary>.*)", re.UNICODE)
+
 NON_MAIN_NS = ['Talk',
                'User',
                'User talk',
@@ -89,6 +92,28 @@ def parse_revs_from_url(url):
         raise ValueError('unparsable url: %r' % (url,))
 
 
+def parse_section_title(summary):
+    """This function tries to extract the section title as it would be
+    automatically generated into the commit message when a user clicks
+    the edit button next to a section. The behavior is basically
+    identical to MediaWiki's, which is to say that it is simple. If
+    the commit message was generated otherwise or manually rewritten,
+    this function will not automatically figure that out.
+
+    Returns a tuple of section_title and "real" commit message
+    (message with the section title part removed).
+    """
+    match = _SECTION_TITLE_RE.match(summary)
+    if not match:
+        return '', summary.strip()
+    match_map = match.groupdict()
+    section_title = match_map['section_title']
+    real_summary = match_map['real_summary']
+    section_title = section_title.strip() if section_title else ''
+    real_summary = real_summary.strip() if real_summary else ''
+    return section_title, real_summary
+
+
 def parse_irc_message(message, ns_map=DEFAULT_NS_MAP):
     ret = PARSE_EDIT_RE.match(message)
     msg_dict = {'is_new': False,
@@ -140,10 +165,13 @@ def parse_irc_message(message, ns_map=DEFAULT_NS_MAP):
     msg_dict.setdefault('user', None)
     msg_dict['is_anon'] = is_ip(msg_dict['user'])
 
-    if msg_dict['summary']:
-        msg_dict['hashtags'] = HASHTAG_RE.findall(msg_dict['summary'])
-        msg_dict['mentions'] = MENTION_RE.findall(msg_dict['summary'])
+    summary = msg_dict['summary']
+    if summary:
+        msg_dict['section'], msg_dict['parsed_summary'] = parse_section_title(summary)
+        msg_dict['hashtags'] = HASHTAG_RE.findall(summary)
+        msg_dict['mentions'] = MENTION_RE.findall(summary)
     else:
+        msg_dict['section'], msg_dict['parsed_summary'] = '', summary
         msg_dict['hashtags'] = []
         msg_dict['mentions'] = []