Processor overhaul of options and validate_query method

common/lib/user_input.py

@@ -1,6 +1,8 @@
 from dateutil.parser import parse as parse_datetime
 from common.lib.exceptions import QueryParametersException
 from werkzeug.datastructures import ImmutableMultiDict
+from pathlib import Path
+import inspect
 import json
 
 import re
@@ -105,7 +107,37 @@ def unknown_option_keys(settings):
         return sorted(set(settings) - UserInput.KNOWN_OPTION_KEYS)
 
     @staticmethod
-    def parse_all(options, input, silently_correct=False, log=None):
+    def describe_source(source):
+        """
+        Say which module some options belong to, and where they are defined.
+        Helper for staticmethod parse_all.
+
+        Look up location of processor through the `get_options` method rather 
+        than the class itself, so that it still points at the right file when a
+        module inherits its options from a parent class instead of defining
+        them itself.
+
+        :param source:  The worker class the options came from, or None
+        :return tuple:  A (name, location) tuple of strings. Either one is
+          empty if it cannot be worked out.
+        """
+        if source is None:
+            return "", ""
+
+        name = getattr(source, "type", "") or getattr(source, "__name__", "")
+
+        try:
+            definition = source.get_options
+            file = Path(inspect.getsourcefile(definition))
+            line = inspect.getsourcelines(definition)[1]
+        except (AttributeError, TypeError, OSError):
+            # the location cannot be read back
+            return name, ""
+
+        return name, "%s:%i" % (file.as_posix(), line)
+
+    @staticmethod
+    def parse_all(options, input, silently_correct=False, log=None, source=None):
         """
         Parse form input for the provided options
 
@@ -137,8 +169,10 @@ def parse_all(options, input, silently_correct=False, log=None):
         behaviour, and tolerant parsing should be opted into deliberately.
         :param log:  Optional logger. When an option's *default* value turns
         out to be invalid (a developer mistake, not user input), a warning is
-        logged here and the default is corrected silently rather than shown to
-        the user.
+        logged here so that it can be fixed.
+        :param source:  Optional worker class the options came from. Only used
+        to say which module a faulty default belongs to, and where that option
+        is defined, so the warning can be acted on without hunting for it.
 
         :return dict:  Sanitised form input
         """
@@ -203,8 +237,13 @@ def parse_all(options, input, silently_correct=False, log=None):
             # reported (naming the option) rather than hidden.
             default_problem = UserInput.validate_default(settings)
             if default_problem and log:
-                log.warning("Option '%s' has an invalid default (%s). "
-                            "Please fix the option definition." % (option, default_problem))
+                source_name, source_location = UserInput.describe_source(source)
+                log.warning("Option '%s'%s has an invalid default (%s). "
+                            "Please fix the option definition%s." % (
+                                option,
+                                " of '%s'" % source_name if source_name else "",
+                                default_problem,
+                                " in %s" % source_location if source_location else ""))
 
             if settings.get("type") == UserInput.OPTION_DATERANGE:
                 # special case, since it combines two inputs

processors/conversion/consolidate_urls.py

@@ -6,7 +6,7 @@
 from ural import is_url
 
 from processors.conversion.extract_urls import ExtractURLs
-from common.lib.exceptions import ProcessorInterruptedException
+from common.lib.exceptions import ProcessorInterruptedException, QueryNeedsExplicitConfirmationException
 from backend.lib.processor import BasicProcessor
 from common.lib.compatibility import Compatibility
 from common.lib.helpers import UserInput, split_urls
@@ -251,6 +251,29 @@ def get_options(cls, parent_dataset=None, config=None):
 
         return options
 
+    @staticmethod
+    def validate_query(query, request, config):
+        """
+        Ask for confirmation before expanding shortened URLs
+
+        Expanding them means contacting each shortening service in turn to ask
+        where its link leads, one request at a time. That takes a while for a
+        large dataset, and it tells those services that the links are being
+        looked up, so it is worth agreeing to first.
+
+        :param dict query:  Query parameters, from client-side.
+        :param request:  Flask request
+        :param ConfigManager|None config:  Configuration reader (context-aware)
+        :return dict:  Safe query parameters
+        """
+        if query.get("expand_urls") and not query.get("frontend-confirm"):
+            raise QueryNeedsExplicitConfirmationException(
+                "Expanding shortened URLs sends a request to each shortening service, one at a time, so this can "
+                "take hours for a large dataset. Those services are also told that the links are being looked up. "
+                "Do you want to continue?")
+
+        return query
+
     def process(self):
         method = self.parameters.get("method", False)
         url_parsing_issues = []

processors/conversion/convert_text.py

@@ -4,7 +4,7 @@
 import re
 import csv
 
-from common.lib.exceptions import ProcessorInterruptedException
+from common.lib.exceptions import ProcessorInterruptedException, QueryParametersException
 from backend.lib.processor import BasicProcessor
 from common.lib.compatibility import Compatibility
 from common.lib.helpers import UserInput
@@ -35,12 +35,13 @@ def get_options(cls, parent_dataset=None, config=None) -> dict:
         options = {
             "columns": {
                 "type": UserInput.OPTION_TEXT,
-                "default": "body",
+                "default": ["body"],
                 "help": "Columns with texts to replace",
             },
             "find": {
                 "type": UserInput.OPTION_TEXT,
                 "default": "",
+                "mandatory": True,
                 "help": "Text to replace",
                 "tooltip": "Multiple values can be replaced; separate with comma.",
             },
@@ -82,11 +83,58 @@ def get_options(cls, parent_dataset=None, config=None) -> dict:
             options["columns"]["type"] = UserInput.OPTION_MULTI
             options["columns"]["inline"] = True
             options["columns"]["options"] = {v: v for v in columns}
-            options["columns"]["default"] = "body" if "body" in columns else sorted(columns,
-                                                                                    key=lambda
-                                                                                        k: "text" in k).pop()
+            # a list: this is a multi-select, so its default is a set of
+            # selected options, not a single one
+            options["columns"]["default"] = ["body"] if "body" in columns else [
+                sorted(columns, key=lambda k: "text" in k).pop()]
         return options
 
+    @staticmethod
+    def compile_find(find, as_regex=False, case_sensitive=False):
+        """
+        Build the regular expression for the text to replace
+
+        Used both to check the text before the job is queued and to do the
+        replacing itself, so that what is checked is exactly what will be used.
+
+        :param str find:  The text to replace, as the user wrote it
+        :param bool as_regex:  Read it as a regular expression? If not, it is
+          escaped, and then any text is safe to use
+        :param bool case_sensitive:  Match capitals exactly?
+        :return re.Pattern|None:  The expression, or None if no text was given
+        :raises QueryParametersException:  If it is not a usable expression
+        """
+        if not find:
+            return None
+
+        flags = 0 if case_sensitive else re.IGNORECASE
+
+        try:
+            if as_regex:
+                return re.compile(find, flags=flags)
+
+            terms = [re.escape(term) for term in find.split(",")]
+            return re.compile(r"(" + "|".join(terms) + r")", flags=flags)
+        except re.error as e:
+            raise QueryParametersException("The text to replace is not a valid regular expression (%s)." % e)
+
+    @staticmethod
+    def validate_query(query, request, config):
+        """
+        Check that the text to replace works as a regular expression
+
+        This only fails when the text is meant to be read as one; it is
+        otherwise escaped before use.
+
+        :param dict query:  Query parameters, from client-side.
+        :param request:  Flask request
+        :param ConfigManager|None config:  Configuration reader (context-aware)
+        :return dict:  Safe query parameters
+        """
+        ConvertText.compile_find(query.get("find"), query.get("as_regex"), query.get("case-sensitive"))
+
+        return query
+
     def process(self):
         """
         Create a generator to iterate through items that can be passed to create either a csv or ndjson.
@@ -108,20 +156,14 @@ def process(self):
         if not replace:
             replace = ""
 
-        case_sensitive = self.parameters.get("case-sensitive", False)
-        kwargs = {}
-        if not case_sensitive:
-            kwargs["flags"] = re.IGNORECASE
-        as_regex = self.parameters.get("as_regex")
-        if not as_regex:
-            find = [re.escape(term) for term in find.split(",")]
+        # the text is checked when the job is queued, but jobs started by
+        # presets and other code do not go through that check, so build it the
+        # same way here and stop if it cannot be used
         try:
-            if not as_regex:
-                regex = re.compile(r"(" + "|".join(find) + r")", **kwargs)
-            else:
-                regex = re.compile(fr"{find}", **kwargs)
-        except re.error:
-            self.dataset.finish_with_error("Invalid regular expression, cannot use as filter")
+            regex = self.compile_find(find, self.parameters.get("as_regex"),
+                                      self.parameters.get("case-sensitive", False))
+        except QueryParametersException as e:
+            self.dataset.finish_with_error(str(e))
             return
 
         save_annotations = self.parameters.get("save_annotations")

processors/conversion/extract_urls.py

@@ -9,7 +9,7 @@
 import requests
 from ural import urls_from_text
 
-from common.lib.exceptions import ProcessorInterruptedException
+from common.lib.exceptions import ProcessorInterruptedException, QueryNeedsExplicitConfirmationException
 from backend.lib.processor import BasicProcessor
 from common.lib.helpers import UserInput
 from common.lib.compatibility import Compatibility
@@ -210,16 +210,41 @@ def get_options(cls, parent_dataset=None, config=None):
             options["columns"]["type"] = UserInput.OPTION_MULTI
             options["columns"]["options"] = {v: v for v in columns}
 
+            # a list: this is a multi-select, so its default is a set of
+            # selected options, not a single one
             if "body" in columns:
-                options["columns"]["default"] = "body"
+                options["columns"]["default"] = ["body"]
             elif text_columns := sorted(columns, key=lambda k: "text" in k):
-                options["columns"]["default"] = text_columns.pop()
+                options["columns"]["default"] = [text_columns.pop()]
             else:
                 # give up, no column we can recognise as text-based
-                options["columns"]["default"] = columns[0]
+                options["columns"]["default"] = [columns[0]]
 
         return options
 
+    @staticmethod
+    def validate_query(query, request, config):
+        """
+        Ask for confirmation before expanding shortened URLs
+
+        Expanding them means contacting each shortening service in turn to ask
+        where its link leads, one request at a time. That takes a while for a
+        large dataset, and it tells those services that the links are being
+        looked up, so it is worth agreeing to first.
+
+        :param dict query:  Query parameters, from client-side.
+        :param request:  Flask request
+        :param ConfigManager|None config:  Configuration reader (context-aware)
+        :return dict:  Safe query parameters
+        """
+        if query.get("expand_urls") and not query.get("frontend-confirm"):
+            raise QueryNeedsExplicitConfirmationException(
+                "Expanding shortened URLs sends a request to each shortening service, one at a time, so this can "
+                "take hours for a large dataset. Those services are also told that the links are being looked up. "
+                "Do you want to continue?")
+
+        return query
+
     def process(self):
         """
         Extract URLs and optionally excand them from URL shorteners

processors/conversion/item_to_annotation.py

@@ -34,6 +34,7 @@ def get_options(cls, parent_dataset=None, config=None) -> dict:
             "columns": {
                 "type": UserInput.OPTION_TEXT,
                 "default": "body",
+                "mandatory": True,
                 "help": "Columns to convert",
             }
         }
@@ -44,6 +45,9 @@ def get_options(cls, parent_dataset=None, config=None) -> dict:
             options["columns"]["type"] = UserInput.OPTION_MULTI
             options["columns"]["inline"] = True
             options["columns"]["options"] = {v: v for v in columns}
+            # a list of columns this parent actually has: the plain "body"
+            # default above is neither a list nor necessarily a real column
+            options["columns"]["default"] = ["body"] if "body" in columns else []
 
         return options
 

processors/conversion/merge_datasets.py

@@ -67,6 +67,7 @@ def get_options(cls, parent_dataset=None, config=None):
         options = {
             "source": {
                 "type": UserInput.OPTION_TEXT_LARGE,
+                "mandatory": True,
                 "help": "Source dataset URLs",
                 "tooltip": "This should be the URL(s) of the result pages of the 4CAT dataset you want to merge with this "
                         "dataset. Note that all datasets need to have the same format! Separate URLs with new lines or "