diff --git a/common/lib/user_input.py b/common/lib/user_input.py index 3bada4489..664916ec3 100644 --- a/common/lib/user_input.py +++ b/common/lib/user_input.py @@ -1,6 +1,8 @@ from dateutil.parser import parse as parse_datetime from common.lib.exceptions import QueryParametersException from werkzeug.datastructures import ImmutableMultiDict +from pathlib import Path +import inspect import json import re @@ -105,7 +107,37 @@ def unknown_option_keys(settings): return sorted(set(settings) - UserInput.KNOWN_OPTION_KEYS) @staticmethod - def parse_all(options, input, silently_correct=False, log=None): + def describe_source(source): + """ + Say which module some options belong to, and where they are defined. + Helper for staticmethod parse_all. + + Look up location of processor through the `get_options` method rather + than the class itself, so that it still points at the right file when a + module inherits its options from a parent class instead of defining + them itself. + + :param source: The worker class the options came from, or None + :return tuple: A (name, location) tuple of strings. Either one is + empty if it cannot be worked out. + """ + if source is None: + return "", "" + + name = getattr(source, "type", "") or getattr(source, "__name__", "") + + try: + definition = source.get_options + file = Path(inspect.getsourcefile(definition)) + line = inspect.getsourcelines(definition)[1] + except (AttributeError, TypeError, OSError): + # the location cannot be read back + return name, "" + + return name, "%s:%i" % (file.as_posix(), line) + + @staticmethod + def parse_all(options, input, silently_correct=False, log=None, source=None): """ Parse form input for the provided options @@ -137,8 +169,10 @@ def parse_all(options, input, silently_correct=False, log=None): behaviour, and tolerant parsing should be opted into deliberately. :param log: Optional logger. When an option's *default* value turns out to be invalid (a developer mistake, not user input), a warning is - logged here and the default is corrected silently rather than shown to - the user. + logged here so that it can be fixed. + :param source: Optional worker class the options came from. Only used + to say which module a faulty default belongs to, and where that option + is defined, so the warning can be acted on without hunting for it. :return dict: Sanitised form input """ @@ -203,8 +237,13 @@ def parse_all(options, input, silently_correct=False, log=None): # reported (naming the option) rather than hidden. default_problem = UserInput.validate_default(settings) if default_problem and log: - log.warning("Option '%s' has an invalid default (%s). " - "Please fix the option definition." % (option, default_problem)) + source_name, source_location = UserInput.describe_source(source) + log.warning("Option '%s'%s has an invalid default (%s). " + "Please fix the option definition%s." % ( + option, + " of '%s'" % source_name if source_name else "", + default_problem, + " in %s" % source_location if source_location else "")) if settings.get("type") == UserInput.OPTION_DATERANGE: # special case, since it combines two inputs diff --git a/processors/conversion/consolidate_urls.py b/processors/conversion/consolidate_urls.py index e3de3d5b6..0f91ff415 100644 --- a/processors/conversion/consolidate_urls.py +++ b/processors/conversion/consolidate_urls.py @@ -6,7 +6,7 @@ from ural import is_url from processors.conversion.extract_urls import ExtractURLs -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, QueryNeedsExplicitConfirmationException from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility from common.lib.helpers import UserInput, split_urls @@ -251,6 +251,29 @@ def get_options(cls, parent_dataset=None, config=None): return options + @staticmethod + def validate_query(query, request, config): + """ + Ask for confirmation before expanding shortened URLs + + Expanding them means contacting each shortening service in turn to ask + where its link leads, one request at a time. That takes a while for a + large dataset, and it tells those services that the links are being + looked up, so it is worth agreeing to first. + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + if query.get("expand_urls") and not query.get("frontend-confirm"): + raise QueryNeedsExplicitConfirmationException( + "Expanding shortened URLs sends a request to each shortening service, one at a time, so this can " + "take hours for a large dataset. Those services are also told that the links are being looked up. " + "Do you want to continue?") + + return query + def process(self): method = self.parameters.get("method", False) url_parsing_issues = [] diff --git a/processors/conversion/convert_text.py b/processors/conversion/convert_text.py index 007c245e6..6f5c67c25 100644 --- a/processors/conversion/convert_text.py +++ b/processors/conversion/convert_text.py @@ -4,7 +4,7 @@ import re import csv -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, QueryParametersException from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility from common.lib.helpers import UserInput @@ -35,12 +35,13 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: options = { "columns": { "type": UserInput.OPTION_TEXT, - "default": "body", + "default": ["body"], "help": "Columns with texts to replace", }, "find": { "type": UserInput.OPTION_TEXT, "default": "", + "mandatory": True, "help": "Text to replace", "tooltip": "Multiple values can be replaced; separate with comma.", }, @@ -82,11 +83,58 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: options["columns"]["type"] = UserInput.OPTION_MULTI options["columns"]["inline"] = True options["columns"]["options"] = {v: v for v in columns} - options["columns"]["default"] = "body" if "body" in columns else sorted(columns, - key=lambda - k: "text" in k).pop() + # a list: this is a multi-select, so its default is a set of + # selected options, not a single one + options["columns"]["default"] = ["body"] if "body" in columns else [ + sorted(columns, key=lambda k: "text" in k).pop()] return options + @staticmethod + def compile_find(find, as_regex=False, case_sensitive=False): + """ + Build the regular expression for the text to replace + + Used both to check the text before the job is queued and to do the + replacing itself, so that what is checked is exactly what will be used. + + :param str find: The text to replace, as the user wrote it + :param bool as_regex: Read it as a regular expression? If not, it is + escaped, and then any text is safe to use + :param bool case_sensitive: Match capitals exactly? + :return re.Pattern|None: The expression, or None if no text was given + :raises QueryParametersException: If it is not a usable expression + """ + if not find: + return None + + flags = 0 if case_sensitive else re.IGNORECASE + + try: + if as_regex: + return re.compile(find, flags=flags) + + terms = [re.escape(term) for term in find.split(",")] + return re.compile(r"(" + "|".join(terms) + r")", flags=flags) + except re.error as e: + raise QueryParametersException("The text to replace is not a valid regular expression (%s)." % e) + + @staticmethod + def validate_query(query, request, config): + """ + Check that the text to replace works as a regular expression + + This only fails when the text is meant to be read as one; it is + otherwise escaped before use. + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + ConvertText.compile_find(query.get("find"), query.get("as_regex"), query.get("case-sensitive")) + + return query + def process(self): """ Create a generator to iterate through items that can be passed to create either a csv or ndjson. @@ -108,20 +156,14 @@ def process(self): if not replace: replace = "" - case_sensitive = self.parameters.get("case-sensitive", False) - kwargs = {} - if not case_sensitive: - kwargs["flags"] = re.IGNORECASE - as_regex = self.parameters.get("as_regex") - if not as_regex: - find = [re.escape(term) for term in find.split(",")] + # the text is checked when the job is queued, but jobs started by + # presets and other code do not go through that check, so build it the + # same way here and stop if it cannot be used try: - if not as_regex: - regex = re.compile(r"(" + "|".join(find) + r")", **kwargs) - else: - regex = re.compile(fr"{find}", **kwargs) - except re.error: - self.dataset.finish_with_error("Invalid regular expression, cannot use as filter") + regex = self.compile_find(find, self.parameters.get("as_regex"), + self.parameters.get("case-sensitive", False)) + except QueryParametersException as e: + self.dataset.finish_with_error(str(e)) return save_annotations = self.parameters.get("save_annotations") diff --git a/processors/conversion/extract_urls.py b/processors/conversion/extract_urls.py index d95600874..18ec94672 100644 --- a/processors/conversion/extract_urls.py +++ b/processors/conversion/extract_urls.py @@ -9,7 +9,7 @@ import requests from ural import urls_from_text -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, QueryNeedsExplicitConfirmationException from backend.lib.processor import BasicProcessor from common.lib.helpers import UserInput from common.lib.compatibility import Compatibility @@ -210,16 +210,41 @@ def get_options(cls, parent_dataset=None, config=None): options["columns"]["type"] = UserInput.OPTION_MULTI options["columns"]["options"] = {v: v for v in columns} + # a list: this is a multi-select, so its default is a set of + # selected options, not a single one if "body" in columns: - options["columns"]["default"] = "body" + options["columns"]["default"] = ["body"] elif text_columns := sorted(columns, key=lambda k: "text" in k): - options["columns"]["default"] = text_columns.pop() + options["columns"]["default"] = [text_columns.pop()] else: # give up, no column we can recognise as text-based - options["columns"]["default"] = columns[0] + options["columns"]["default"] = [columns[0]] return options + @staticmethod + def validate_query(query, request, config): + """ + Ask for confirmation before expanding shortened URLs + + Expanding them means contacting each shortening service in turn to ask + where its link leads, one request at a time. That takes a while for a + large dataset, and it tells those services that the links are being + looked up, so it is worth agreeing to first. + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + if query.get("expand_urls") and not query.get("frontend-confirm"): + raise QueryNeedsExplicitConfirmationException( + "Expanding shortened URLs sends a request to each shortening service, one at a time, so this can " + "take hours for a large dataset. Those services are also told that the links are being looked up. " + "Do you want to continue?") + + return query + def process(self): """ Extract URLs and optionally excand them from URL shorteners diff --git a/processors/conversion/item_to_annotation.py b/processors/conversion/item_to_annotation.py index bd91871a6..65c907052 100644 --- a/processors/conversion/item_to_annotation.py +++ b/processors/conversion/item_to_annotation.py @@ -34,6 +34,7 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "columns": { "type": UserInput.OPTION_TEXT, "default": "body", + "mandatory": True, "help": "Columns to convert", } } @@ -44,6 +45,9 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: options["columns"]["type"] = UserInput.OPTION_MULTI options["columns"]["inline"] = True options["columns"]["options"] = {v: v for v in columns} + # a list of columns this parent actually has: the plain "body" + # default above is neither a list nor necessarily a real column + options["columns"]["default"] = ["body"] if "body" in columns else [] return options diff --git a/processors/conversion/merge_datasets.py b/processors/conversion/merge_datasets.py index a2022b57e..0184020ec 100644 --- a/processors/conversion/merge_datasets.py +++ b/processors/conversion/merge_datasets.py @@ -67,6 +67,7 @@ def get_options(cls, parent_dataset=None, config=None): options = { "source": { "type": UserInput.OPTION_TEXT_LARGE, + "mandatory": True, "help": "Source dataset URLs", "tooltip": "This should be the URL(s) of the result pages of the 4CAT dataset you want to merge with this " "dataset. Note that all datasets need to have the same format! Separate URLs with new lines or " diff --git a/processors/filtering/lexical_filter.py b/processors/filtering/lexical_filter.py index 4c0129f65..b184a18e8 100644 --- a/processors/filtering/lexical_filter.py +++ b/processors/filtering/lexical_filter.py @@ -5,6 +5,7 @@ from processors.filtering.base_filter import BaseFilter from common.lib.compatibility import Compatibility +from common.lib.exceptions import QueryParametersException from common.lib.helpers import UserInput __author__ = "Stijn Peeters" @@ -45,6 +46,7 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "lexicon-custom": { "type": UserInput.OPTION_TEXT, "default": "", + "mandatory": True, "help": "Custom word list (separate with commas)" }, "as_regex": { @@ -66,6 +68,59 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: } } + @staticmethod + def compile_lexicon(words, as_regex=False, case_sensitive=False): + """ + Build a word list's regular expression + + Used both to check the words before the job is queued and to do the + filtering itself, so that what is checked is exactly what will be used. + Accepts the words as the user wrote them or as a collection + + The words are joined into a single expression, so they have to be + checked together: a word can be a valid expression on its own but not + once joined with the others. + + :param words: The words, comma-separated or as a collection + :param bool as_regex: Read the words as regular expressions? If not, + they are escaped, and then any text is safe to use + :param bool case_sensitive: Match capitals exactly? + :return re.Pattern|None: The expression, or None if there are no words + :raises QueryParametersException: If it is not a usable expression + """ + if isinstance(words, str): + words = words.split(",") + + phrases = [word.strip() for word in words if word and word.strip()] + if not phrases: + return None + + if not as_regex: + phrases = [re.escape(phrase) for phrase in phrases] + + try: + return re.compile(r"\b(" + "|".join(phrases) + r")\b", flags=0 if case_sensitive else re.IGNORECASE) + except re.error as e: + raise QueryParametersException("The word list is not a valid regular expression (%s)." % e) + + @staticmethod + def validate_query(query, request, config): + """ + Check that the word list works as a regular expression + + This only fails when the words are meant to be read as regular + expressions; they are otherwise escaped before use. + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + LexicalFilter.compile_lexicon( + query.get("lexicon-custom", ""), query.get("as_regex"), query.get("case-sensitive")) + + return query + def filter_items(self): """ Create a generator to iterate through items that can be passed to create either a csv or ndjson. Use @@ -93,27 +148,19 @@ def filter_items(self): [word.strip() for word in custom_lexicon.split(",") if word.strip()]) lexicons[custom_id] |= custom_lexicon - # compile into regex for quick matching + # compile into regex for quick matching. the words are checked when the + # job is queued, but jobs started by presets and other code do not go + # through that check, so build them the same way here lexicon_regexes = {} for lexicon_id in lexicons: if not lexicons[lexicon_id]: continue - if not self.parameters.get("as_regex"): - phrases = [re.escape(term) for term in lexicons[lexicon_id] if term] - else: - phrases = [term for term in lexicons[lexicon_id] if term] - try: - if not case_sensitive: - lexicon_regexes[lexicon_id] = re.compile( - r"\b(" + "|".join(phrases) + r")\b", - flags=re.IGNORECASE) - else: - lexicon_regexes[lexicon_id] = re.compile( - r"\b(" + "|".join(phrases) + r")\b") - except re.error: - self.dataset.finish_with_error("Invalid regular expression, cannot use as filter") + lexicon_regexes[lexicon_id] = self.compile_lexicon( + lexicons[lexicon_id], self.parameters.get("as_regex"), case_sensitive) + except QueryParametersException as e: + self.dataset.finish_with_error(str(e)) return # now for the real deal diff --git a/processors/filtering/unique_filter.py b/processors/filtering/unique_filter.py index 8a82dfc47..ffd5d9391 100644 --- a/processors/filtering/unique_filter.py +++ b/processors/filtering/unique_filter.py @@ -109,6 +109,7 @@ def get_options(cls, parent_dataset=None, config=None): options = { "columns": { "type": UserInput.OPTION_TEXT, + "mandatory": True, "help": "Columns to consider for uniqueness", "inline": True, "default": "body" @@ -138,6 +139,8 @@ def get_options(cls, parent_dataset=None, config=None): columns = parent_dataset.get_columns() options["columns"]["type"] = UserInput.OPTION_MULTI options["columns"]["options"] = {v: v for v in columns} - options["columns"]["default"] = "body" if "body" in columns else "" + # a list: this is a multi-select, so its default is a set of + # selected options, not a single one + options["columns"]["default"] = ["body"] if "body" in columns else [] return options diff --git a/processors/machine_learning/audio_to_text.py b/processors/machine_learning/audio_to_text.py index ed4078ba0..bc1d6046e 100644 --- a/processors/machine_learning/audio_to_text.py +++ b/processors/machine_learning/audio_to_text.py @@ -9,7 +9,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility from common.lib.dmi_service_manager import DmiServiceManager, DmiServiceManagerException, DsmOutOfMemory -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, QueryNeedsExplicitConfirmationException from common.lib.user_input import UserInput from common.lib.item_mapping import MappedItem @@ -255,11 +255,33 @@ def get_options(cls, parent_dataset=None, config=None): "help": "OpenAI API key", "tooltip": "Can be created on platform.openapi.com", "requires": "model_host==openai", - "sensitive": True + "sensitive": True, + "mandatory": True } return options + @staticmethod + def validate_query(query, request, config): + """ + Ask for confirmation before audio is sent to OpenAI + + Sending audio to OpenAI shares data with a third party and is likely to + cost money, so ask the user to confirm that first. That the API key is + actually filled in is handled by the option's `mandatory` setting. + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + if query.get("model_host") == "openai" and not query.get("frontend-confirm"): + raise QueryNeedsExplicitConfirmationException( + "Your audio files will be sent to OpenAI for processing. This shares your data with a third " + "party and is likely to incur costs. Do you want to continue?") + + return query + def process(self): """ This takes a zipped set of audio files and uses a Whisper docker image to identify speech and convert to text, @@ -318,7 +340,9 @@ def process(self): return else: - # Check for API key if using OpenAI models + # the key is also checked when the job is queued, but jobs started + # by presets and `next` steps skip that check, and a site-wide key + # can be removed between queueing and running api_key = self.parameters.get("api_key") if not api_key: api_key = self.config.get("api.openai.api_key") diff --git a/processors/machine_learning/clarifai_api.py b/processors/machine_learning/clarifai_api.py index 34ebe67cf..718eba570 100644 --- a/processors/machine_learning/clarifai_api.py +++ b/processors/machine_learning/clarifai_api.py @@ -59,7 +59,8 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "type": UserInput.OPTION_TEXT, "help": "Images to process (0 = all)", "cache": True, - "sensitive": True, + "coerce_type": int, + "min": 0, "default": 0 }, "api_key": { @@ -67,6 +68,7 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "help": "API Key", "cache": True, "sensitive": True, + "mandatory": True, "tooltip": "The API key for your Clarifai account. You'll need to go to clarifai.com and create a new " "project to generate a key." }, diff --git a/processors/machine_learning/clip_categorize_images.py b/processors/machine_learning/clip_categorize_images.py index 99aa084d1..9d3e7299e 100644 --- a/processors/machine_learning/clip_categorize_images.py +++ b/processors/machine_learning/clip_categorize_images.py @@ -103,6 +103,7 @@ def get_options(cls, parent_dataset=None, config=None): "type": UserInput.OPTION_TEXT, "help": "Categories (comma seperated list)", "default": "", + "mandatory": True, "tooltip": "The CLIP model will estimate the probability an image belongs to every category, adding up to 100% across categories. It is quite robust and can accept proper nouns, some celebrities, as well as understand syntax such as \"animal\" vs \"not animal\"" }, } diff --git a/processors/machine_learning/generate_images.py b/processors/machine_learning/generate_images.py index 273d0835c..a9b964c23 100644 --- a/processors/machine_learning/generate_images.py +++ b/processors/machine_learning/generate_images.py @@ -92,7 +92,8 @@ def get_options(cls, parent_dataset=None, config=None): }, "prompt-column": { "type": UserInput.OPTION_TEXT, - "default": False, + "default": "", + "mandatory": True, "help": "Dataset field containing prompt", "tooltip": "Prompts will be truncated to 70 characters" }, diff --git a/processors/machine_learning/google_vision_api.py b/processors/machine_learning/google_vision_api.py index 1f6f6eeea..2ef8308e9 100644 --- a/processors/machine_learning/google_vision_api.py +++ b/processors/machine_learning/google_vision_api.py @@ -59,11 +59,14 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "amount": { "type": UserInput.OPTION_TEXT, "help": "Images to process (0 = all)", + "coerce_type": int, + "min": 0, "default": 0 }, "api_key": { "type": UserInput.OPTION_TEXT, "sensitive": True, + "mandatory": True, "help": "API Key", "tooltip": "The API Key for the Google API account you want to query with. You can generate and find this" "key on console.cloud.google.com. You also need to enable billing and Vision API." diff --git a/processors/machine_learning/perspective.py b/processors/machine_learning/perspective.py index 4b88035d3..e9f9e774a 100644 --- a/processors/machine_learning/perspective.py +++ b/processors/machine_learning/perspective.py @@ -72,7 +72,8 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "default": "", "help": "Google API key", "tooltip": "Can be created on console.cloud.google.com", - "sensitive": True + "sensitive": True, + "mandatory": True } return options diff --git a/processors/metrics/count_posts.py b/processors/metrics/count_posts.py index e30cabe1c..a525a54cf 100644 --- a/processors/metrics/count_posts.py +++ b/processors/metrics/count_posts.py @@ -62,10 +62,12 @@ def get_options(cls, parent_dataset=None, config=None): options["column"]["inline"] = True options["column"]["tooltip"] = "Choose one. If multiple are selected, the first will be used." options["column"]["options"] = {v: v for v in columns if "time" in v or "date" in v or "created" in v} + # a list: this is a multi-select, so its default is a set of + # selected options, not a single one options["column"]["default"] = ( - "timestamp" + ["timestamp"] if "timestamp" in columns - else sorted(columns, key=lambda k: "time" in k).pop() + else [sorted(columns, key=lambda k: "time" in k).pop()] ) return options diff --git a/processors/metrics/rank_attribute.py b/processors/metrics/rank_attribute.py index bc11298cf..7e83cdaee 100644 --- a/processors/metrics/rank_attribute.py +++ b/processors/metrics/rank_attribute.py @@ -9,6 +9,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility +from common.lib.exceptions import QueryParametersException from common.lib.helpers import UserInput, convert_to_int, get_interval_descriptor __author__ = "Stijn Peeters" @@ -95,7 +96,9 @@ def get_options(cls, parent_dataset=None, config=None): "top": { "type": UserInput.OPTION_TEXT, "default": 25, - "help": "Limit to this amount of results" + "coerce_type": int, + "min": 0, + "help": "Limit to this amount of results (0 = no limit)" }, "top-style": { "type": UserInput.OPTION_CHOICE, @@ -146,6 +149,40 @@ def get_options(cls, parent_dataset=None, config=None): return options + @staticmethod + def compile_filter(item_filter): + """ + Build the item filter's regular expression + + Used both to check the filter before the job is queued and to do the + filtering itself, so that what is checked is exactly what will be used. + + :param str item_filter: The filter as the user wrote it + :return re.Pattern|None: The expression, or None if no filter was given + :raises QueryParametersException: If it is not a usable expression + """ + if not item_filter: + return None + + try: + return re.compile(".*" + item_filter + ".*") + except (re.error, TypeError) as e: + raise QueryParametersException("The item filter is not a valid regular expression (%s)." % e) + + @staticmethod + def validate_query(query, request, config): + """ + Check that the item filter is a usable regular expression + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + AttributeRanker.compile_filter(query.get("filter")) + + return query + def process(self): """ Reads a CSV file, counts occurences of chosen values over all posts, @@ -169,16 +206,17 @@ def process(self): to_lowercase = self.parameters.get("to-lowercase", True) self.include_missing_data = self.parameters.get("count_missing") + # the filter is checked when the job is queued, but jobs started by + # presets and other code do not go through that check, so build it the + # same way here and stop if it cannot be used try: - if self.parameters.get("filter"): - filter = re.compile(".*" + self.parameters.get("filter") + ".*") - else: - filter = None - negate_filter = self.parameters.get("negate-filter", False) - except (TypeError, re.error): - self.dataset.finish_with_error("Could not complete: regular expression invalid") + filter = self.compile_filter(self.parameters.get("filter")) + except QueryParametersException as e: + self.dataset.finish_with_error(str(e)) return + negate_filter = self.parameters.get("negate-filter", False) + # we need to be able to order the values later, chronologically, so use # and OrderedDict; all frequencies go into this variable items = OrderedDict() diff --git a/processors/metrics/vocabulary_overtime.py b/processors/metrics/vocabulary_overtime.py index eae9b2712..d54eb9a6f 100644 --- a/processors/metrics/vocabulary_overtime.py +++ b/processors/metrics/vocabulary_overtime.py @@ -5,6 +5,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility +from common.lib.exceptions import QueryParametersException from common.lib.helpers import UserInput, get_interval_descriptor __author__ = "Stijn Peeters" @@ -80,6 +81,24 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: } } + @staticmethod + def validate_query(query, request, config): + """ + Check that there are words to count + + Either vocabulary or vocabulary-custom must be provided, otherwise the processor will not know what to count. + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + if not query.get("vocabulary") and not query.get("vocabulary-custom", "").strip(): + raise QueryParametersException( + "Choose at least one vocabulary, or provide your own words to count.") + + return query + def process(self): """ Reads a CSV file, counts occurrences of chosen values over all posts, diff --git a/processors/metrics/youtube_metadata.py b/processors/metrics/youtube_metadata.py index 6dacf0fe3..5a6195b84 100644 --- a/processors/metrics/youtube_metadata.py +++ b/processors/metrics/youtube_metadata.py @@ -122,9 +122,10 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: columns = parent_dataset.get_columns() options["columns"]["type"] = UserInput.OPTION_MULTI options["columns"]["options"] = {v: v for v in columns} - options["columns"]["default"] = "extracted_urls" if "extracted_urls" in columns else sorted(columns, - key=lambda - k: "text" in k).pop() + # a list: this is a multi-select, so its default is a set of + # selected options, not a single one + options["columns"]["default"] = ["extracted_urls"] if "extracted_urls" in columns else [ + sorted(columns, key=lambda k: "text" in k).pop()] api_key = config.get("api.youtube.key") if not api_key: options["key"] = { @@ -132,7 +133,8 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "default": "", "help": "YouTube API key", "tooltip": "Can be created on https://developers.google.com/youtube/v3", - "sensitive": True + "sensitive": True, + "mandatory": True } return options diff --git a/processors/networks/clarifai_bipartite_network.py b/processors/networks/clarifai_bipartite_network.py index 35bbdcd2f..45ac74563 100644 --- a/processors/networks/clarifai_bipartite_network.py +++ b/processors/networks/clarifai_bipartite_network.py @@ -43,6 +43,9 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "min_confidence": { "type": UserInput.OPTION_TEXT, "default": 0.5, + "coerce_type": float, + "min": 0, + "max": 1, "help": "Min confidence", "tooltip": "Value between 0 and 1; confidence required before the annotation is included. Note that the" \ "confidence is not known for all annotation types (these will be included with confidence '-1'" \ diff --git a/processors/networks/google_vision_bipartite_network.py b/processors/networks/google_vision_bipartite_network.py index aef0f3c19..678e7e95e 100644 --- a/processors/networks/google_vision_bipartite_network.py +++ b/processors/networks/google_vision_bipartite_network.py @@ -44,6 +44,9 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "min_confidence": { "type": UserInput.OPTION_TEXT, "default": 0.5, + "coerce_type": float, + "min": 0, + "max": 1, "help": "Min confidence", "tooltip": "Value between 0 and 1; confidence required before the annotation is included. Note that the" \ "confidence is not known for all annotation types (these will be included with confidence '-1'" \ diff --git a/processors/networks/google_vision_network.py b/processors/networks/google_vision_network.py index 9ddf0da8d..54cb2010c 100644 --- a/processors/networks/google_vision_network.py +++ b/processors/networks/google_vision_network.py @@ -44,6 +44,9 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "min_confidence": { "type": UserInput.OPTION_TEXT, "default": 0.5, + "coerce_type": float, + "min": 0, + "max": 1, "help": "Min confidence", "tooltip": "Value between 0 and 1; confidence required before the annotation is included. Note that the" \ "confidence is not known for all annotation types (these will be included with confidence '-1'" \ diff --git a/processors/networks/hash_similarity_network.py b/processors/networks/hash_similarity_network.py index 845c91814..3f69c8ad5 100644 --- a/processors/networks/hash_similarity_network.py +++ b/processors/networks/hash_similarity_network.py @@ -8,7 +8,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility -from common.lib.exceptions import ProcessorException +from common.lib.exceptions import ProcessorException, QueryParametersException from common.lib.helpers import UserInput @@ -71,6 +71,21 @@ def get_options(cls, parent_dataset=None, config=None): return options + @staticmethod + def validate_query(query, request, config): + """ + Check that the hashes and the IDs come from different columns + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + if query.get("descriptor_column") == query.get("choice_column"): + raise QueryParametersException("The hashes and the IDs must come from two different columns.") + + return query + def process(self): """ Takes a list of bit hashes and compares them. Then makes network file. diff --git a/processors/networks/two-column-network.py b/processors/networks/two-column-network.py index f79cb6f85..959c740d4 100644 --- a/processors/networks/two-column-network.py +++ b/processors/networks/two-column-network.py @@ -6,6 +6,7 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility +from common.lib.exceptions import QueryParametersException from common.lib.helpers import UserInput, get_interval_descriptor import networkx as nx @@ -143,6 +144,26 @@ def get_options(cls, parent_dataset=None, config=None): return options + @staticmethod + def validate_query(query, request, config): + """ + Check that the chosen columns can produce any edges at all + + If column a and b are the same, loops or split-comma must also be True. + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + same_column = query.get("column-a") == query.get("column-b") + if same_column and not query.get("split-comma") and not query.get("allow-loops"): + raise QueryParametersException( + "Using one column for both sides of the network only produces edges from a value to itself. " + "Choose a second column, or enable splitting values by comma or allowing loops.") + + return query + def process(self): """ This takes a 4CAT results file as input, and creates a network file diff --git a/processors/presets/annotate-images.py b/processors/presets/annotate-images.py index 57acbc55e..cb83fcec5 100644 --- a/processors/presets/annotate-images.py +++ b/processors/presets/annotate-images.py @@ -41,6 +41,8 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "amount": { "type": UserInput.OPTION_TEXT, "help": "Images to process (0 = all)", + "coerce_type": int, + "min": 0, "default": 10, "tooltip": "Setting this to 0 (process all images) is NOT recommended unless you have infinite Google API" " credit." @@ -48,6 +50,7 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "api_key": { "type": UserInput.OPTION_TEXT, "sensitive": True, + "mandatory": True, "help": "API Key", "tooltip": "The API Key for your Google API account. You can generate and find this " "key on the API dashboard." diff --git a/processors/presets/monthly-histogram.py b/processors/presets/monthly-histogram.py index a67464b5a..d434b4901 100644 --- a/processors/presets/monthly-histogram.py +++ b/processors/presets/monthly-histogram.py @@ -22,8 +22,8 @@ class MonthlyHistogramCreator(ProcessorPreset): @classmethod def get_options(cls, parent_dataset=None, config=None): count_options = CountPosts.get_options(parent_dataset=parent_dataset, config=config) - if "all" in count_options["timeframe"]: - # Cannot graph overall counts (or rather it would be a single bar) + # Cannot graph overall counts (or rather it would be a single bar) + if "all" in count_options["timeframe"]["options"]: count_options["timeframe"]["options"].pop("all") return count_options diff --git a/processors/presets/neologisms.py b/processors/presets/neologisms.py index a0aa7d849..8c40586b2 100644 --- a/processors/presets/neologisms.py +++ b/processors/presets/neologisms.py @@ -39,6 +39,7 @@ def get_options(cls, parent_dataset=None, config=None): }, "columns": { "type": UserInput.OPTION_TEXT, + "mandatory": True, "help": "Column(s) from which to extract neologisms", "tooltip": "Each enabled column will be treated as a separate item to tokenise. Columns must contain text." }, @@ -50,7 +51,9 @@ def get_options(cls, parent_dataset=None, config=None): options["columns"]["options"] = {v: v for v in columns} default_options = [default for default in ["body", "text", "subject"] if default in columns] if default_options: - options["columns"]["default"] = default_options.pop(0) + # a list: this is a multi-select, so its default is a set of + # selected options, not a single one + options["columns"]["default"] = default_options[:1] return options diff --git a/processors/statistics/confusion_matrix.py b/processors/statistics/confusion_matrix.py index 00e4c521e..7f62a3e73 100644 --- a/processors/statistics/confusion_matrix.py +++ b/processors/statistics/confusion_matrix.py @@ -34,6 +34,7 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: options = { "columns": { "type": UserInput.OPTION_TEXT, + "mandatory": True, "help": "Column to use for true and predicted values", "inline": True, "default": "", diff --git a/processors/statistics/descriptive_statistics.py b/processors/statistics/descriptive_statistics.py index 6156ecfac..0fed7d2cc 100644 --- a/processors/statistics/descriptive_statistics.py +++ b/processors/statistics/descriptive_statistics.py @@ -33,6 +33,7 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: options = { "columns": { "type": UserInput.OPTION_MULTI, + "mandatory": True, "help": "Columns to analyze", "options": {}, "default": [], diff --git a/processors/text-analysis/collocations.py b/processors/text-analysis/collocations.py index e777dc499..b36e76dc2 100644 --- a/processors/text-analysis/collocations.py +++ b/processors/text-analysis/collocations.py @@ -8,6 +8,7 @@ import operator from nltk.collocations import TrigramCollocationFinder, BigramCollocationFinder +from common.lib.exceptions import QueryParametersException from common.lib.helpers import UserInput from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility @@ -87,11 +88,15 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "min_frequency": { "type": UserInput.OPTION_TEXT, "default": 1, + "coerce_type": int, + "min": 1, "help": "Minimum frequency of co-words occurrences" }, "max_output": { "type": UserInput.OPTION_TEXT, "default": 0, + "coerce_type": int, + "min": 0, "help": "Maximum number of top co-words to extract (0 = all)" }, "save_annotations": { @@ -103,6 +108,27 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: } } + @staticmethod + def validate_query(query, request, config): + """ + Check that the requested word groups fit in the word window + + Asking for groups of more words than the window holds cannot be done, + and used to quietly produce smaller groups instead - so someone who + asked for three-word groups received two-word ones without being told. + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + if int(query.get("n_size", 2)) > int(query.get("window_size", 2)): + raise QueryParametersException( + "The window size must be at least as large as the number of co-words, " + "since a window of that many words cannot contain a larger group.") + + return query + def process(self): """ Unzips token sets, vectorises them and zips them again. diff --git a/processors/text-analysis/generate_embeddings.py b/processors/text-analysis/generate_embeddings.py index ea352895b..b6bfe8e08 100644 --- a/processors/text-analysis/generate_embeddings.py +++ b/processors/text-analysis/generate_embeddings.py @@ -93,6 +93,8 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "min_count": { "type": UserInput.OPTION_TEXT, "default": 5, + "coerce_type": int, + "min": 1, "help": "Minimum word occurrence", "tooltip": "How often a word should occur in the corpus to be included" }, diff --git a/processors/text-analysis/similar_words.py b/processors/text-analysis/similar_words.py index ad8f7db8a..6b9acfa5e 100644 --- a/processors/text-analysis/similar_words.py +++ b/processors/text-analysis/similar_words.py @@ -8,7 +8,7 @@ from common.lib.helpers import UserInput, convert_to_int, convert_to_float from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, QueryNeedsExplicitConfirmationException __author__ = "Sal Hagen" __credits__ = ["Sal Hagen", "Stijn Peeters"] @@ -45,6 +45,7 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: return { "words": { "type": UserInput.OPTION_TEXT, + "mandatory": True, "help": "Words", "tooltip": "Separate with commas." }, @@ -59,7 +60,10 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "type": UserInput.OPTION_TEXT, "help": "Similarity threshold", "tooltip": "Decimal value between 0 and 1; only words with a higher similarity score than this will be included", - "default": "0.25" + "coerce_type": float, + "min": -1, # Technically -1 is possible, but may not make sense in context + "max": 1, + "default": 0.25 }, "crawl_depth": { "type": UserInput.OPTION_CHOICE, @@ -69,6 +73,34 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: } } + @staticmethod + def validate_query(query, request, config): + """ + Ask for confirmation before a very wide crawl + + Each level looks up the neighbours of every word the level before it + found, so the work multiplies with each one: a few dozen words at the + first level become tens of thousands by the third. That is a long run + and a very large result, so ask first when the numbers chosen make it + likely. Words that turn up more than once are only looked at once, so + the real total is lower than this estimate. + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + words = [word.strip() for word in query.get("words", "").split(",") if word.strip()] + estimate = len(words) * int(query.get("num-words", 10)) ** int(query.get("crawl_depth", 1)) + + if estimate > 10000 and not query.get("frontend-confirm"): + raise QueryNeedsExplicitConfirmationException( + "With this crawl depth and this many similar words, up to around %s words could be looked up per " + "model, which can take a long time and produce a very large result. Do you want to continue?" % ( + f"{estimate:,}")) + + return query + def process(self): """ This takes previously generated Word2Vec models and uses them to find diff --git a/processors/text-analysis/tf_idf.py b/processors/text-analysis/tf_idf.py index 46d5c90a6..e96add2fb 100644 --- a/processors/text-analysis/tf_idf.py +++ b/processors/text-analysis/tf_idf.py @@ -7,12 +7,14 @@ import pandas as pd import itertools +from common.lib.exceptions import QueryParametersException from common.lib.helpers import UserInput, convert_to_int from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility from sklearn.feature_extraction.text import TfidfVectorizer from gensim.models import TfidfModel +from gensim.models.tfidfmodel import resolve_weights from gensim import corpora __author__ = "Sal Hagen" @@ -94,7 +96,7 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "min": 0, "max": 10000, "help": "[scikit-learn] Ignore terms that appear in more than this amount of documents", - "tooltip": "Useful for getting more specific terms per document. Leaving empty means terms may appear in all documents. For instance, if you have 12 monthly documents and insert 10 here, terms may not appear in 11 or 12 months.", + "tooltip": "Useful for getting more specific terms per document. For instance, if you have 12 monthly documents and insert 10 here, terms may not appear in 11 or 12 months. Leaving as '0' means terms may appear in all documents.", "requires": "library==scikit-learn" }, "n_size": { @@ -114,6 +116,43 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: } } + @staticmethod + def validate_query(query, request, config): + """ + Check the occurrence range and the SMART parameters + + Asking for words that appear in at least X but at most fewer than X + documents describes no word at all, and leaves nothing to calculate + with. Zero means "no maximum", so it is not a lower bound to compare + against. + + Each option only exists for the library it belongs to, so a + missing key here means the user was offered the other one. + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + # min and max only exist if sklearn chosen + if "min_occurrences" in query and "max_occurrences" in query: + maximum = query["max_occurrences"] + if maximum and query["min_occurrences"] > maximum: + raise QueryParametersException( + "Words cannot appear in both more than %s and fewer than %s documents. Lower the minimum, " + "raise the maximum, or set the maximum to 0 for no upper limit." % ( + query["min_occurrences"], maximum)) + + # smartirs only exists if gensim chosen. gensim decides what it accepts, + # so ask it rather than keeping a second copy of the rules here. + if "smartirs" in query: + try: + resolve_weights(query["smartirs"]) + except ValueError as e: + raise QueryParametersException("These SMART parameters cannot be used: %s." % e) + + return query + def process(self): """ Unzips and appends tokens to fetch and write a tf-idf matrix diff --git a/processors/text-analysis/tokenise.py b/processors/text-analysis/tokenise.py index 577900efa..46597cc3a 100644 --- a/processors/text-analysis/tokenise.py +++ b/processors/text-analysis/tokenise.py @@ -71,6 +71,7 @@ def get_options(cls, parent_dataset=None, config=None): options = { "columns": { "type": UserInput.OPTION_MULTI, + "mandatory": True, "help": "Column(s) to tokenise", "tooltip": "Each enabled column will be treated as a separate item to tokenise. Columns must contain text." }, @@ -190,7 +191,9 @@ def get_options(cls, parent_dataset=None, config=None): options["columns"]["options"] = {v: v for v in columns} default_options = [default for default in ["body", "text", "subject"] if default in columns] if default_options: - options["columns"]["default"] = default_options.pop(0) + # a list: this is a multi-select, so its default is a set of + # selected options, not a single one + options["columns"]["default"] = default_options[:1] return options diff --git a/processors/text-analysis/top_vectors.py b/processors/text-analysis/top_vectors.py index a04d33d3c..9d2929028 100644 --- a/processors/text-analysis/top_vectors.py +++ b/processors/text-analysis/top_vectors.py @@ -43,6 +43,8 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "top": { "type": UserInput.OPTION_TEXT, "default": 25, + "coerce_type": int, + "min": 1, "help": "Cut-off for top list" }, "top-style": { diff --git a/processors/text-analysis/topic_modeling.py b/processors/text-analysis/topic_modeling.py index d9070ab67..94ca5dae0 100644 --- a/processors/text-analysis/topic_modeling.py +++ b/processors/text-analysis/topic_modeling.py @@ -5,7 +5,7 @@ from common.lib.helpers import UserInput from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility -from common.lib.exceptions import ProcessorInterruptedException +from common.lib.exceptions import ProcessorInterruptedException, QueryParametersException import json import pickle @@ -80,7 +80,7 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: }, "max_df": { "type": UserInput.OPTION_TEXT, - "min": 0.0, + "min": 0.0, # real min is > 0 but 0.00000000...000001?; we'll check for that in validate_query() "max": 1.0, "default": 0.8, "help": "Maximum document frequency", @@ -88,6 +88,30 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: } } + @staticmethod + def validate_query(query, request, config): + """ + Check that the document frequency bounds work together + + Both bounds are already limited to numbers between 0 and 1 when the + form input is parsed, but they can still contradict each other, which + would otherwise only surface as an error halfway through the run. + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + if query.get("max_df") == 0: + raise QueryParametersException("The maximum document frequency must be above zero, or every token " + "would be ignored.") + + if "min_df" in query and "max_df" in query and query["min_df"] > query["max_df"]: + raise QueryParametersException("The minimum document frequency cannot be higher than the maximum " + "document frequency, or every token would be ignored.") + + return query + def process(self): """ Unzips token sets and builds topic models for each one. Model data is diff --git a/processors/visualisation/download_images.py b/processors/visualisation/download_images.py index bd13b21c7..2e4a46163 100644 --- a/processors/visualisation/download_images.py +++ b/processors/visualisation/download_images.py @@ -129,18 +129,19 @@ def get_options(cls, parent_dataset=None, config=None): columns = parent_dataset.get_columns() options["columns"]["type"] = UserInput.OPTION_MULTI options["columns"]["options"] = {v: v for v in columns} - # Pick a good default + # Pick a good default. a list: this is a multi-select, so its + # default is a set of selected options, not a single one if "image_url" in columns: - options["columns"]["default"] = "image_url" + options["columns"]["default"] = ["image_url"] elif any("image" in (col or "").lower() for col in columns): # Any image will do image_cols = sorted( [col for col in columns if "image" in (col or "").lower()], key=lambda c: (len(c), c.lower()), ) - options["columns"]["default"] = image_cols[0] if image_cols else "body" + options["columns"]["default"] = [image_cols[0]] if image_cols else ["body"] else: - options["columns"]["default"] = "body" + options["columns"]["default"] = ["body"] return options diff --git a/processors/visualisation/histwords.py b/processors/visualisation/histwords.py index 93846d0cd..c910a013d 100644 --- a/processors/visualisation/histwords.py +++ b/processors/visualisation/histwords.py @@ -67,6 +67,7 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: return { "words": { "type": UserInput.OPTION_TEXT, + "mandatory": True, "help": "Word(s)", "tooltip": "Nearest neighbours for these words will be charted, and the position of the words will be highlighted" }, diff --git a/processors/visualisation/isoviz.py b/processors/visualisation/isoviz.py index a1199abda..88ff24fe7 100644 --- a/processors/visualisation/isoviz.py +++ b/processors/visualisation/isoviz.py @@ -79,6 +79,9 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "complete": { "type": UserInput.OPTION_TEXT, "default": 0, + "coerce_type": int, + "min": 0, + "max": 100, "help": "Data completeness required", "tooltip": "At least this % of intervals should be present for an item to be graphed; 0 to disable" }, diff --git a/processors/visualisation/word-cloud.py b/processors/visualisation/word-cloud.py index be922a0a5..c9fb9db3d 100644 --- a/processors/visualisation/word-cloud.py +++ b/processors/visualisation/word-cloud.py @@ -7,7 +7,8 @@ from backend.lib.processor import BasicProcessor from common.lib.compatibility import Compatibility -from common.lib.helpers import UserInput +from common.lib.exceptions import QueryParametersException +from common.lib.helpers import UserInput, convert_to_int __author__ = "Sal Hagen" __credits__ = ["Sal Hagen"] @@ -62,11 +63,35 @@ def get_options(cls, parent_dataset=None, config=None): "max_words": { "type": UserInput.OPTION_TEXT, "default": 200, + "min": 1, "help": "Max words to show" } } return options + @staticmethod + def validate_query(query, request, config): + """ + Check that a word and count column were chosen + + The column choices default to an empty value when the parent dataset + has no obvious word and count columns, so the form can be submitted + with an empty choice. Catch that here so the user can fix it in the + form, instead of the processor failing after it has started. + + :param dict query: Query parameters, from client-side. + :param request: Flask request + :param ConfigManager|None config: Configuration reader (context-aware) + :return dict: Safe query parameters + """ + if "word_column" in query and not query.get("word_column"): + raise QueryParametersException("Select the column containing the words for the word cloud.") + + if "count_column" in query and not query.get("count_column"): + raise QueryParametersException("Select the column containing the word counts.") + + return query + def process(self): """ Render an SVG histogram/bar chart using a previous frequency analysis @@ -78,13 +103,15 @@ def process(self): word_column = self.parameters.get("word_column") count_column = self.parameters.get("count_column") to_lower = self.parameters.get("to_lower") - try: - max_words = int(self.parameters.get("max_words")) - except (ValueError, TypeError): - max_words = self.parameters["max_words"]["default"] + # form input is checked when the job is queued, but presets and other + # code can start this processor with unchecked values, so make sure + # the amount is a usable number here as well + max_words = max(1, convert_to_int(self.parameters.get("max_words"), 200)) self.dataset.update_status("Extracting words and counts.") + # also checked when the job is queued, but kept for jobs started by + # presets and other code, which skip that check if not word_column: self.dataset.finish_with_error("Please set a valid word column.") return diff --git a/processors/visualisation/word-trees.py b/processors/visualisation/word-trees.py index f1b493570..562e5b766 100644 --- a/processors/visualisation/word-trees.py +++ b/processors/visualisation/word-trees.py @@ -8,7 +8,6 @@ from backend.lib.processor import BasicProcessor from common.lib.helpers import UserInput, convert_to_int, get_4cat_canvas -from common.lib.exceptions import QueryParametersException from common.lib.compatibility import Compatibility from nltk.tokenize import word_tokenize, TweetTokenizer @@ -256,6 +255,7 @@ def get_options(cls, parent_dataset=None, config=None): "query": { "type": UserInput.OPTION_TEXT, "default": "", + "mandatory": True, "help": "Word tree root query", "tooltip": "Enter a text here to serve as the root of the word tree. The context of this query will be " "visualised as a tree graph. You can use wildcards: 'politic*' will match 'politician' and " @@ -341,7 +341,9 @@ def get_options(cls, parent_dataset=None, config=None): default for default in ["body", "text", "subject"] if default in columns ] if default_options: - options["columns"]["default"] = default_options.pop(0) + # a list: this is a multi-select, so its default is a set of + # selected options, not a single one + options["columns"]["default"] = default_options[:1] return options @@ -451,23 +453,6 @@ def process(self): return self.dataset.finish(len(list(PreOrderIter(root)))) - @staticmethod - def validate_query(query, request, config): - """ - Validate input - - Checks if everything needed is filled in. - - :param query: - :param request: - :param config: - :return: - """ - if not query.get("query", "").strip(): - raise QueryParametersException("Query cannot be empty.") - - return query - def build_tree(self, query: str) -> tuple[TreeNode, int]: """ Build a tree of text fragments to then visualise diff --git a/processors/visualisation/youtube_imagewall.py b/processors/visualisation/youtube_imagewall.py index 94e9c969d..a710c6236 100644 --- a/processors/visualisation/youtube_imagewall.py +++ b/processors/visualisation/youtube_imagewall.py @@ -51,6 +51,8 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "max_amount": { "type": UserInput.OPTION_TEXT, "default": 0, + "coerce_type": int, + "min": 0, "help": "Only use this amount of thumbnails (0 = all)" }, "category_overlay": { diff --git a/processors/visualisation/youtube_thumbnails.py b/processors/visualisation/youtube_thumbnails.py index 111cd6a43..e8026a4bc 100644 --- a/processors/visualisation/youtube_thumbnails.py +++ b/processors/visualisation/youtube_thumbnails.py @@ -48,7 +48,8 @@ def get_options(cls, parent_dataset=None, config=None) -> dict: "default": "", "help": "YouTube API key", "tooltip": "Can be created on https://developers.google.com/youtube/v3", - "sensitive": True + "sensitive": True, + "mandatory": True } } diff --git a/tests/test_modules.py b/tests/test_modules.py index c08ecfa03..356ef617b 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -187,6 +187,31 @@ def mock_dataset(mock_dataset_database, fourcat_modules): yield dataset +@pytest.fixture +def mock_dataset_with_columns(mock_dataset_database, fourcat_modules): + """ + A parent dataset that has columns + + Processors routinely build their options differently when the parent has + columns: a plain text field becomes a list of columns to pick from, and a + default is chosen out of those columns. get_columns() returns nothing when + there is no result file to read, as with the mock dataset above, so that + whole branch of get_options() never runs - and the options it produces go + unchecked. This parent reports columns so those options are built too. + """ + from common.lib.dataset import DataSet + + columns = ["id", "thread_id", "timestamp", "body", "subject", "author", "image_url", "video_hash"] + with patch.object(DataSet, "refresh_owners", return_value=None), \ + patch.object(DataSet, "get_parent", return_value=MagicMock(type="test_parent")): + dataset = DataSet(key="test_dataset", db=mock_dataset_database, modules=fourcat_modules) + # give the columns to this dataset only, rather than to every DataSet: + # the column-less parent is used in the same test and has to keep + # reporting none, or both parents would be the same thing + dataset.get_columns = lambda annotation_columns=True: columns + yield dataset + + @pytest.mark.dependency(depends=["test_module_collector"]) def test_processors(logger, fourcat_modules, mock_job, mock_job_queue, mock_dataset, mock_database, mock_basic_config): """ @@ -254,7 +279,8 @@ def test_processors(logger, fourcat_modules, mock_job, mock_job_queue, mock_data @pytest.mark.dependency(depends=["test_module_collector"]) -def test_option_default_validity(logger, fourcat_modules, mock_job, mock_job_queue, mock_dataset, mock_database, mock_basic_config): +def test_option_default_validity(logger, fourcat_modules, mock_job, mock_job_queue, mock_dataset, + mock_dataset_with_columns, mock_database, mock_basic_config): """ Every option's declared default must be valid under that option's own rules. A bad default -- a number outside its own min/max, a coerce_type @@ -262,6 +288,12 @@ def test_option_default_validity(logger, fourcat_modules, mock_job, mock_job_que unparseable JSON -- is a mistake in the option definition, not user input, and should never reach a user. Catch those here instead. + Options are checked for a parent dataset with and without columns, because + a processor commonly offers different options for each: without columns a + field may be plain text, and with them the same field becomes a list of + columns to choose from, with a default picked out of those columns. Only + checking one of the two leaves the other's options unchecked. + Uses UserInput.validate_default. Membership of choice/multi defaults in option lists that are built from the parent dataset at runtime is intentionally not checked (it cannot be verified out of context). @@ -269,23 +301,25 @@ def test_option_default_validity(logger, fourcat_modules, mock_job, mock_job_que from common.lib.user_input import UserInput problems = [] - for processor_name, processor_class in fourcat_modules.processors.items(): - try: - options = processor_class.get_options(parent_dataset=mock_dataset, config=mock_basic_config) - except Exception: - # a failing get_options is test_processors' concern, not this test's - continue - - if not isinstance(options, dict): - continue + for parent_description, parent in (("no columns", mock_dataset), + ("with columns", mock_dataset_with_columns)): + for processor_name, processor_class in fourcat_modules.processors.items(): + try: + options = processor_class.get_options(parent_dataset=parent, config=mock_basic_config) + except Exception: + # a failing get_options is test_processors' concern, not this test's + continue - for option_name, settings in options.items(): - if not isinstance(settings, dict): + if not isinstance(options, dict): continue - problem = UserInput.validate_default(settings) - if problem: - logger.error(f"{processor_name} option '{option_name}': {problem}") - problems.append(f"{processor_name}.{option_name}: {problem}") + + for option_name, settings in options.items(): + if not isinstance(settings, dict): + continue + problem = UserInput.validate_default(settings) + if problem: + logger.error(f"{processor_name} option '{option_name}' (parent {parent_description}): {problem}") + problems.append(f"{processor_name}.{option_name} (parent {parent_description}): {problem}") if problems: pytest.fail("The following option defaults are invalid:\n" + "\n".join(problems)) @@ -294,32 +328,40 @@ def test_option_default_validity(logger, fourcat_modules, mock_job, mock_job_que @pytest.mark.dependency(depends=["test_module_collector"]) -def test_option_keys_known(logger, fourcat_modules, mock_job, mock_job_queue, mock_dataset, mock_database, mock_basic_config): +def test_option_keys_known(logger, fourcat_modules, mock_job, mock_job_queue, mock_dataset, + mock_dataset_with_columns, mock_database, mock_basic_config): """ Every key in an option's settings dictionary must be one the framework understands. An unknown key (e.g. 'coerce' instead of 'coerce_type') is silently ignored at parse time, so the setting it was meant to express quietly has no effect. Catch those typos here. + Checked for a parent dataset with and without columns, since a processor + may offer different options for each (see test_option_default_validity). + See UserInput.KNOWN_OPTION_KEYS for the recognised set. """ from common.lib.user_input import UserInput problems = [] - for processor_name, processor_class in fourcat_modules.processors.items(): - try: - options = processor_class.get_options(parent_dataset=mock_dataset, config=mock_basic_config) - except Exception: - continue + for parent_description, parent in (("no columns", mock_dataset), + ("with columns", mock_dataset_with_columns)): + for processor_name, processor_class in fourcat_modules.processors.items(): + try: + options = processor_class.get_options(parent_dataset=parent, config=mock_basic_config) + except Exception: + continue - if not isinstance(options, dict): - continue + if not isinstance(options, dict): + continue - for option_name, settings in options.items(): - unknown = UserInput.unknown_option_keys(settings) - if unknown: - logger.error(f"{processor_name} option '{option_name}': unknown key(s) {unknown}") - problems.append(f"{processor_name}.{option_name}: unknown key(s) {', '.join(unknown)}") + for option_name, settings in options.items(): + unknown = UserInput.unknown_option_keys(settings) + if unknown: + logger.error(f"{processor_name} option '{option_name}' (parent {parent_description}): " + f"unknown key(s) {unknown}") + problems.append(f"{processor_name}.{option_name} (parent {parent_description}): " + f"unknown key(s) {', '.join(unknown)}") if problems: pytest.fail("The following options have unrecognised keys:\n" + "\n".join(problems)) diff --git a/webtool/views/api_tool.py b/webtool/views/api_tool.py index 55fb3f3b1..d475449db 100644 --- a/webtool/views/api_tool.py +++ b/webtool/views/api_tool.py @@ -459,7 +459,7 @@ def queue_dataset(): # just in case try: # first sanitise values - sanitised_query = UserInput.parse_all(search_worker.get_options(None, g.config), request.form, silently_correct=False, log=g.log) + sanitised_query = UserInput.parse_all(search_worker.get_options(None, g.config), request.form, silently_correct=False, log=g.log, source=search_worker) # then validate for this particular datasource sanitised_query = {"frontend-confirm": has_confirm, **sanitised_query} @@ -1212,6 +1212,7 @@ def queue_processor(key=None, processor=None): request.form, silently_correct=False, log=g.log, + source=processor_worker, ) sanitised_query["frontend-confirm"] = bool(request.form.get("frontend-confirm", False))