From b5eebb881694a4b0131181a5772d50d1027a8810 Mon Sep 17 00:00:00 2001
From: Lukas Kaufmann <Lukas.Kaufmann@relaxdays.de>
Date: Sun, 17 Jan 2021 19:16:14 +0100
Subject: [PATCH 1/4] detecting easy comments in php (copied from c)

---
 README.md                                     |   1 +
 comment_parser/comment_parser.py              |   3 +
 comment_parser/parsers/php_parser.py          |  65 +++++++++++
 .../parsers/tests/php_parser_test.py          | 102 ++++++++++++++++++
 4 files changed, 171 insertions(+)
 create mode 100644 comment_parser/parsers/php_parser.py
 create mode 100644 comment_parser/parsers/tests/php_parser_test.py

diff --git a/README.md b/README.md
index 8d2408f..66cfe28 100644
--- a/README.md
+++ b/README.md
@@ -154,6 +154,7 @@ twine upload dist/*
 | Ruby        | text/x-ruby              |
 | Shell       | text/x-shellscript       |
 | XML         | text/xml                 |
+| PHP         | text/x-php               |
 
 And more to come!
 
diff --git a/comment_parser/comment_parser.py b/comment_parser/comment_parser.py
index 04d5ba6..d8564db 100755
--- a/comment_parser/comment_parser.py
+++ b/comment_parser/comment_parser.py
@@ -11,6 +11,7 @@
   Javascript
   Ruby
   XML
+  PHP
 
 Dependencies:
   python-magic: pip install python-magic (optional)
@@ -32,6 +33,7 @@
 from comment_parser.parsers import python_parser
 from comment_parser.parsers import ruby_parser
 from comment_parser.parsers import shell_parser
+from comment_parser.parsers import php_parser
 
 MIME_MAP = {
     'application/javascript': js_parser,  # Javascript
@@ -46,6 +48,7 @@
     'text/x-ruby': ruby_parser,  # Ruby
     'text/x-shellscript': shell_parser,  # Unix shell
     'text/xml': html_parser,  # XML
+    'text/x-php': php_parser,  # PHP
 }
 
 
diff --git a/comment_parser/parsers/php_parser.py b/comment_parser/parsers/php_parser.py
new file mode 100644
index 0000000..f0d368c
--- /dev/null
+++ b/comment_parser/parsers/php_parser.py
@@ -0,0 +1,65 @@
+#!/usr/bin/python
+"""This module provides methods for parsing comments from PHP.
+
+Works with:
+  PHP
+"""
+
+import re
+from bisect import bisect_left
+from comment_parser.parsers import common
+
+
+def extract_comments(code):
+  """Extracts a list of comments from the given PHP source code.
+
+  Comments are represented with the Comment class found in the common module.
+  PHP comments come in two forms, single and multi-line comments.
+    - Single-line comments begin with '//' or '#' and continue to the end of line.
+    - Multi-line comments begin with '/*' and end with '*/' and can span
+      multiple lines of code. If a multi-line comment does not terminate
+      before EOF is reached, then an exception is raised.
+
+  Note that this doesn't take language-specific preprocessor directives into
+  consideration.
+
+  Args:
+    code: String containing code to extract comments from.
+  Returns:
+    Python list of common.Comment in the order that they appear in the code.
+  Raises:
+    common.UnterminatedCommentError: Encountered an unterminated multi-line
+      comment.
+  """
+  pattern = r"""
+    (?P<literal> ([\"'])((?:\\\2|(?:(?!\2)).)*)(\2)) |
+    (?P<single> (?://|\#)(?P<single_content>.*)?$) |
+    (?P<multi> /\*(?P<multi_content>(.|\n)*?)?\*/) |
+    (?P<error> /\*(.*)?)
+  """
+
+  compiled = re.compile(pattern, re.VERBOSE | re.MULTILINE)
+
+  lines_indexes = []
+  for match in re.finditer(r"$", code, re.M):
+    lines_indexes.append(match.start())
+
+  comments = []
+  for match in compiled.finditer(code):
+    kind = match.lastgroup
+
+    start_character = match.start()
+    line_no = bisect_left(lines_indexes, start_character)
+
+    if kind == "single":
+      comment_content = match.group("single_content")
+      comment = common.Comment(comment_content, line_no + 1)
+      comments.append(comment)
+    elif kind == "multi":
+      comment_content = match.group("multi_content")
+      comment = common.Comment(comment_content, line_no + 1, multiline=True)
+      comments.append(comment)
+    elif kind == "error":
+      raise common.UnterminatedCommentError()
+
+  return comments
diff --git a/comment_parser/parsers/tests/php_parser_test.py b/comment_parser/parsers/tests/php_parser_test.py
new file mode 100644
index 0000000..1451657
--- /dev/null
+++ b/comment_parser/parsers/tests/php_parser_test.py
@@ -0,0 +1,102 @@
+#!/usr/bin/python
+"""Tests for comment_parser.parsers.c_parser.py"""
+
+import unittest
+from comment_parser.parsers import common
+from comment_parser.parsers import php_parser
+
+
+class PHPParserTest(unittest.TestCase):
+
+  def testSimpleSingleLineComment(self):
+    code = """<?php
+    // this is a comment
+    echo "Hello World";"""
+    comments = php_parser.extract_comments(code)
+    expected = [common.Comment(" this is a comment", 2, multiline=False)]
+    self.assertEqual(comments, expected)
+
+  def testOtherSingleLineComment(self):
+    code = """<?php
+    # this is a comment
+    echo "Hello World";"""
+    comments = php_parser.extract_comments(code)
+    expected = [common.Comment(" this is a comment", 2, multiline=False)]
+    self.assertEqual(comments, expected)
+
+  def testSingleLineCommentInStringLiteral(self):
+    code = '''<?php
+    echo "// this is not a comment";'''
+    comments = php_parser.extract_comments(code)
+    self.assertEqual(comments, [])
+
+  def testMultiLineComment(self):
+    code = '''<?php
+    /* multiline\ncomment */'''
+    comments = php_parser.extract_comments(code)
+    expected = [common.Comment(' multiline\ncomment ', 2, multiline=True)]
+    self.assertEqual(comments, expected)
+
+  def testMultiLineCommentWithStars(self):
+    code = """<?php
+    /***************/"""
+    comments = php_parser.extract_comments(code)
+    expected = [common.Comment("*************", 2, multiline=True)]
+    self.assertEqual(comments, expected)
+
+  def testMultiLineCommentInStringLiteral(self):
+    code = '''<?php
+    echo "/* This is not a\\nmultiline comment */";'''
+    comments = php_parser.extract_comments(code)
+    self.assertEqual(comments, [])
+
+  def testMultiLineCommentUnterminated(self):
+    code = '''<?php
+    $a = 1; /* Unterminated\\n comment'''
+    self.assertRaises(common.UnterminatedCommentError,
+                      php_parser.extract_comments, code)
+
+  def testMultipleMultilineComments(self):
+    code = '''<?php
+    /* abc */ /* 123 */'''
+    comments = php_parser.extract_comments(code)
+    expected = [
+        common.Comment(' abc ', 2, multiline=True),
+        common.Comment(' 123 ', 2, multiline=True),
+    ]
+    self.assertEqual(comments, expected)
+
+  def testStringThenComment(self):
+    code = '''<?php
+    echo "" /* "abc */;'''
+    comments = php_parser.extract_comments(code)
+    expected = [
+        common.Comment(' "abc ', 2, multiline=True),
+    ]
+    self.assertEqual(comments, expected)
+
+  def testCommentStartInsideEscapedQuotesInStringLiteral(self):
+    # TODO(#27): Re-enable test.
+    # code = r'" \" /* \" "'
+    # comments = c_parser.extract_comments(code)
+    # self.assertEqual(comments, [])
+    pass
+
+  def testStringEscapedBackslashCharacter(self):
+    code = r'''<?php
+    echo "\\"; # This wouldn't be a comment, if the second " is misinterpreted as escaped
+    '''
+    comments = php_parser.extract_comments(code)
+    self.assertEqual(comments, [])
+
+  def testCommentedOtherComment(self):
+    code = '''<?php
+    //# double comment'''
+    comments = php_parser.extract_comments(code)
+    self.assertEqual(comments, [common.Comment('# double comment', 2)])
+
+  def testOtherCommentedComment(self):
+    code = '''<?php
+    #// double comment'''
+    comments = php_parser.extract_comments(code)
+    self.assertEqual(comments, [common.Comment('// double comment', 2)])

From d3ccd5d00a0e541fef7605525321b1afee17b707 Mon Sep 17 00:00:00 2001
From: Lukas Kaufmann <Lukas.Kaufmann@relaxdays.de>
Date: Sun, 17 Jan 2021 23:18:11 +0100
Subject: [PATCH 2/4] php tags and multiline string support

---
 comment_parser/parsers/php_parser.py          |  4 +-
 .../parsers/tests/php_parser_test.py          | 38 +++++++++++++++++++
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/comment_parser/parsers/php_parser.py b/comment_parser/parsers/php_parser.py
index f0d368c..e743731 100644
--- a/comment_parser/parsers/php_parser.py
+++ b/comment_parser/parsers/php_parser.py
@@ -32,7 +32,7 @@ def extract_comments(code):
       comment.
   """
   pattern = r"""
-    (?P<literal> ([\"'])((?:\\\2|(?:(?!\2)).)*)(\2)) |
+    (?P<literal> (?:([\"'])((?:\\\2|(?:(?!\2)).|\n)*)(\2))|\?>((?!<\?php\s).|\n)*<\?php\s) |
     (?P<single> (?://|\#)(?P<single_content>.*)?$) |
     (?P<multi> /\*(?P<multi_content>(.|\n)*?)?\*/) |
     (?P<error> /\*(.*)?)
@@ -45,7 +45,7 @@ def extract_comments(code):
     lines_indexes.append(match.start())
 
   comments = []
-  for match in compiled.finditer(code):
+  for match in compiled.finditer("?>\n"+code+"\n<?php "):
     kind = match.lastgroup
 
     start_character = match.start()
diff --git a/comment_parser/parsers/tests/php_parser_test.py b/comment_parser/parsers/tests/php_parser_test.py
index 1451657..e0ba120 100644
--- a/comment_parser/parsers/tests/php_parser_test.py
+++ b/comment_parser/parsers/tests/php_parser_test.py
@@ -100,3 +100,41 @@ def testOtherCommentedComment(self):
     #// double comment'''
     comments = php_parser.extract_comments(code)
     self.assertEqual(comments, [common.Comment('// double comment', 2)])
+
+  def testNoPhpTag(self):
+    code = '''#// double comment'''
+    comments = php_parser.extract_comments(code)
+    self.assertEqual(comments, [])
+
+  def testCommentedPhpTag(self):
+    code = '''<?php
+    # ?>
+    /* Wouldn't be a commend if commented php end tag was misinterpreted */'''
+    comments = php_parser.extract_comments(code)
+    expected = [
+        common.Comment(' ?>', 2, multiline=False),
+        common.Comment(" Wouldn't be a commend if commented php end tag was misinterpreted ", 3, multiline=True)
+    ]
+    self.assertEqual(comments, expected)
+
+  def testCommentsOutsidePhpTag(self):
+    code = '''<?php echo "Hi";
+    ?>
+    // This is no comment <?php
+    // But this is'''
+    comments = php_parser.extract_comments(code)
+    expected = [
+        common.Comment(' But this is', 4, multiline=False)
+    ]
+    self.assertEqual(comments, expected)
+
+  def testMultilineString(self):
+    code = '''<?php echo "Hi
+    // Multi
+    # Line
+    ?>
+    <?php
+    /* String */
+    ";'''
+    comments = php_parser.extract_comments(code)
+    self.assertEqual(comments, [])

From be61e979d046edf49e4320c7430c4a830a6fa265 Mon Sep 17 00:00:00 2001
From: Lukas Kaufmann <Lukas.Kaufmann@relaxdays.de>
Date: Sun, 17 Jan 2021 23:37:47 +0100
Subject: [PATCH 3/4] Support for HereDocStrings

---
 comment_parser/parsers/php_parser.py          |  4 +--
 .../parsers/tests/php_parser_test.py          | 34 ++++++++++++++++---
 2 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/comment_parser/parsers/php_parser.py b/comment_parser/parsers/php_parser.py
index e743731..72089cf 100644
--- a/comment_parser/parsers/php_parser.py
+++ b/comment_parser/parsers/php_parser.py
@@ -32,7 +32,7 @@ def extract_comments(code):
       comment.
   """
   pattern = r"""
-    (?P<literal> (?:([\"'])((?:\\\2|(?:(?!\2)).|\n)*)(\2))|\?>((?!<\?php\s).|\n)*<\?php\s) |
+    (?P<literal> (?:([\"'])((?:\\\2|(?:(?!\2)).|\n)*)(\2))|\?>((?!<\?php\s).|\n)*<\?php\s|<<<('?)(([a-zA-Z0-9_]|[^\x00-\x7F])([a-zA-Z0-9_]|[^\x00-\x7F])*)\6((?!^\7;?$)(.|\n))*^\7;?$) |
     (?P<single> (?://|\#)(?P<single_content>.*)?$) |
     (?P<multi> /\*(?P<multi_content>(.|\n)*?)?\*/) |
     (?P<error> /\*(.*)?)
@@ -45,7 +45,7 @@ def extract_comments(code):
     lines_indexes.append(match.start())
 
   comments = []
-  for match in compiled.finditer("?>\n"+code+"\n<?php "):
+  for match in compiled.finditer("?>\n" + code + "\n<?php "):
     kind = match.lastgroup
 
     start_character = match.start()
diff --git a/comment_parser/parsers/tests/php_parser_test.py b/comment_parser/parsers/tests/php_parser_test.py
index e0ba120..92fee51 100644
--- a/comment_parser/parsers/tests/php_parser_test.py
+++ b/comment_parser/parsers/tests/php_parser_test.py
@@ -113,7 +113,10 @@ def testCommentedPhpTag(self):
     comments = php_parser.extract_comments(code)
     expected = [
         common.Comment(' ?>', 2, multiline=False),
-        common.Comment(" Wouldn't be a commend if commented php end tag was misinterpreted ", 3, multiline=True)
+        common.Comment(
+            " Wouldn't be a commend if commented php end tag was misinterpreted ",
+            3,
+            multiline=True)
     ]
     self.assertEqual(comments, expected)
 
@@ -123,9 +126,7 @@ def testCommentsOutsidePhpTag(self):
     // This is no comment <?php
     // But this is'''
     comments = php_parser.extract_comments(code)
-    expected = [
-        common.Comment(' But this is', 4, multiline=False)
-    ]
+    expected = [common.Comment(' But this is', 4, multiline=False)]
     self.assertEqual(comments, expected)
 
   def testMultilineString(self):
@@ -138,3 +139,28 @@ def testMultilineString(self):
     ";'''
     comments = php_parser.extract_comments(code)
     self.assertEqual(comments, [])
+
+  def testHereDocString(self):
+    code = '''<?php echo <<<Aä_0
+    // Multi "
+    # Line
+    ?>
+    <?php
+    /* String */"\nAä_0;'''
+    comments = php_parser.extract_comments(code)
+    self.assertEqual(comments, [])
+
+  def testFakeHereDocString(self):
+    code = '''<?php echo "<<<Aä_0
+    // Multi ";
+    # Line
+    ?>
+    \nAä_0;\n
+    <?php
+    /* String */"'''
+    comments = php_parser.extract_comments(code)
+    expect = [
+        common.Comment(" Line", 3, False),
+        common.Comment(" String ", 9, True)
+    ]
+    self.assertEqual(comments, expect)

From 27635a3437a1218474f46b07ac7894febf8a331b Mon Sep 17 00:00:00 2001
From: Lukas Kaufmann <Lukas.Kaufmann@relaxdays.de>
Date: Mon, 18 Jan 2021 10:20:49 +0100
Subject: [PATCH 4/4] Fixed wrong line number for some multiline comments

---
 comment_parser/parsers/php_parser.py            | 13 ++++++++++---
 comment_parser/parsers/tests/php_parser_test.py |  6 ++++++
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/comment_parser/parsers/php_parser.py b/comment_parser/parsers/php_parser.py
index 72089cf..3d93bad 100644
--- a/comment_parser/parsers/php_parser.py
+++ b/comment_parser/parsers/php_parser.py
@@ -39,13 +39,17 @@ def extract_comments(code):
   """
 
   compiled = re.compile(pattern, re.VERBOSE | re.MULTILINE)
+  # The regex recognizes stuff between ?> and <?php as literal
+  # The following wrapping sets the expectation to be outside of php tags at the start
+  # and deals with the state, where the php tag is not open at the end of the file
+  code = "?>\n" + code + "\n<?php "
 
   lines_indexes = []
   for match in re.finditer(r"$", code, re.M):
     lines_indexes.append(match.start())
 
   comments = []
-  for match in compiled.finditer("?>\n" + code + "\n<?php "):
+  for match in compiled.finditer(code):
     kind = match.lastgroup
 
     start_character = match.start()
@@ -53,11 +57,14 @@ def extract_comments(code):
 
     if kind == "single":
       comment_content = match.group("single_content")
-      comment = common.Comment(comment_content, line_no + 1)
+      comment = common.Comment(comment_content,
+                               line_no)  # Line number is increased by wrapping
       comments.append(comment)
     elif kind == "multi":
       comment_content = match.group("multi_content")
-      comment = common.Comment(comment_content, line_no + 1, multiline=True)
+      comment = common.Comment(
+          comment_content, line_no,
+          multiline=True)  # Line number is increased by wrapping
       comments.append(comment)
     elif kind == "error":
       raise common.UnterminatedCommentError()
diff --git a/comment_parser/parsers/tests/php_parser_test.py b/comment_parser/parsers/tests/php_parser_test.py
index 92fee51..289c06a 100644
--- a/comment_parser/parsers/tests/php_parser_test.py
+++ b/comment_parser/parsers/tests/php_parser_test.py
@@ -164,3 +164,9 @@ def testFakeHereDocString(self):
         common.Comment(" String ", 9, True)
     ]
     self.assertEqual(comments, expect)
+
+  def testMultilineCommentWithEmptyFirstLine(self):
+    code = '<?php\n/*\n| Comment Roll\n*/\nint main(){return 0;}'
+    comments = php_parser.extract_comments(code)
+    expected = [common.Comment('\n| Comment Roll\n', 2, True)]
+    self.assertEqual(comments, expected)