Source code for raider.plugins

# Copyright (C) 2021 DigeeX
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

"""Plugins used as inputs/outputs in Flows.
"""

import json
import logging
import os
import re
from base64 import b64encode
from typing import Callable, Dict, Optional

import hy
import requests
from bs4 import BeautifulSoup

from raider.utils import hy_dict_to_python, match_tag, parse_json_filter


[docs]class Plugin: """Parent class for all plugins. Each Plugin class inherits from here. "get_value" function should be called when extracting the value from the plugin, which will then be stored in the "value" attribute. Attributes: name: A string used as an identifier for the Plugin. function: A function which will be called to extract the "value" of the Plugin when used as an input in a Flow. The function should set self.value and also return it. value: A string containing the Plugin's output value to be used as input in the HTTP request. flags: An integer containing the flags that define the Plugin's behaviour. For now only NEEDS_USERDATA and NEEDS_RESPONSE is supported. If NEEDS_USERDATA is set, the plugin will get its value from the user's data, which will be sent to the function defined here. If NEEDS_RESPONSE is set, the Plugin will extract its value from the HTTP response instead. """ # Plugin flags NEEDS_USERDATA = 0x01 NEEDS_RESPONSE = 0x02 def __init__( self, name: str, function: Callable[..., Optional[str]], flags: int = 0, value: Optional[str] = None, ) -> None: """Initializes a Plugin object. Creates a Plugin object, holding a "function" defining how to extract the "value". Args: name: A string with the unique identifier of the Plugin. function: A Callable function that will be used to extract the Plugin's value. value: A string with the extracted value from the Plugin. flags: An integer containing the flags that define the Plugin's behaviour. For now only NEEDS_USERDATA and NEEDS_RESPONSE is supported. If NEEDS_USERDATA is set, the plugin will get its value from the user's data, which will be sent to the function defined here. If NEEDS_RESPONSE is set, the Plugin will extract its value from the HTTP response instead. """ self.name = name self.function = function self.value: Optional[str] = value self.flags = flags def extract_value( self, response: Optional[requests.models.Response], ) -> None: """Extracts the value of the Plugin from the HTTP response. If NEEDS_RESPONSE flag is set, the Plugin will extract its value upon receiving the HTTP response, and store it inside the "value" attribute. Args: response: An requests.models.Response object with the HTTP response. """ if self.needs_response: output = self.function(response) if output: self.value = output logging.debug( "Found ouput %s = %s", self.name, self.value, ) else: logging.warning("Couldn't extract output: %s", str(self.name)) def get_value( self, userdata: Dict[str, str], ) -> Optional[str]: """Gets the value from the Plugin. Depending on the Plugin's flags, extract and return its value. Args: userdata: A dictionary with the user specific data. """ if self.needs_userdata: self.value = self.function(userdata) elif not self.needs_response: self.value = self.function() return self.value @property def needs_userdata(self) -> bool: """Returns True if the NEEDS_USERDATA flag is set.""" return bool(self.flags & self.NEEDS_USERDATA) @property def needs_response(self) -> bool: """Returns True if the NEEDS_RESPONSE flag is set.""" return bool(self.flags & self.NEEDS_RESPONSE)
[docs]class Regex(Plugin): """Plugin to extract something using regular expressions. This plugin will match the regex provided, and extract the value inside the matched group, which by default is the first one. A group is the string that matched inside the brackets. For example if the regular expression is: "accessToken":"([^"]+)" and the text to match it against contains: "accessToken":"0123456789abcdef" then only the string "0123456789abcdef" will be extracted and saved in the "value" attribute. Attributes: regex: A string containing the regular expression to be matched. extract: An integer with the group number that needs to be extracted. """ def __init__(self, name: str, regex: str, extract: int = 0) -> None: """Initializes the Regex Plugin. Creates a Regex Plugin with the given regular expression, and extracts the matched group given in the "extract" argument, or the first matching group if not specified. Args: name: A string with the name of the Plugin. regex: A string containing the regular expression to be matched. extract: An optional integer with the number of the group to be extracted. By default the first group will be assumed. """ super().__init__( name=name, function=self.extract_regex, flags=self.NEEDS_RESPONSE, ) self.regex = regex self.extract = extract
[docs] def extract_regex( self, response: requests.models.Response ) -> Optional[str]: """Extracts defined regular expression from a text. Given a text to be searched for matches, return the string inside the group defined in "extract" or the first group if it's undefined. Args: text: A string containing the text to be searched for matches. Returns: A string with the match from the extracted group. Returns None if there are no matches. """ matches = re.search(self.regex, response.text) if matches: groups = matches.groups() self.value = groups[self.extract] logging.debug("Regex %s: %s", self.name, str(self.value)) else: logging.warning( "Regex %s not found in the response body", self.name ) return self.value
[docs] def __str__(self) -> str: """Returns a string representation of the Plugin.""" return "Regex:" + self.regex + ":" + str(self.extract)
[docs]class Html(Plugin): """Plugin to extract something from an HTML tag. This Plugin will find the HTML "tag" containing the specified "attributes" and store the "extract" attribute of the matched tag in its "value" attribute. Attributes: tag: A string defining the HTML tag to look for. attributes: A dictionary with attributes matching the desired HTML tag. The keys in the dictionary are strings matching the tag's attributes, and the values are treated as regular expressions, to help match tags that don't have a static value. extract: A string defining the HTML tag's attribute that needs to be extracted and stored inside "value". """ def __init__( self, name: str, tag: str, attributes: Dict[hy.HyKeyword, str], extract: str, ) -> None: """Initializes the Html Plugin. Creates a Html Plugin with the given "tag" and "attributes". Stores the "extract" attribute in the plugin's "value". Args: name: A string with the name of the Plugin. tag: A string with the HTML tag to look for. attributes: A hy dictionary with the attributes to look inside HTML tags. The values of dictionary elements are treated as regular expressions. extract: A string with the HTML tag attribute that needs to be extracted and stored in the Plugin's object. """ super().__init__( name=name, function=self.extract_html_tag, flags=self.NEEDS_RESPONSE, ) self.tag = tag self.attributes = hy_dict_to_python(attributes) self.extract = extract
[docs] def extract_html_tag( self, response: requests.models.Response ) -> Optional[str]: """Extract data from an HTML tag. Given the HTML text, parses it, iterates through the tags, and find the one matching the attributes. Then it stores the matched "value" and returns it. Args: text: A string containing the HTML text to be processed. Returns: A string with the match as defined in the Plugin. Returns None if there are no matches. """ soup = BeautifulSoup(response.text, "html.parser") matches = soup.find_all(self.tag) for item in matches: if match_tag(item, self.attributes): self.value = item.attrs.get(self.extract) logging.debug("Html filter %s: %s", self.name, str(self.value)) return self.value
[docs] def __str__(self) -> str: """Returns a string representation of the Plugin.""" return ( "Html:" + self.tag + ":" + str(self.attributes) + ":" + str(self.extract) )
[docs]class Json(Plugin): """Plugin to extract a field from JSON. The "extract" attribute is used to specify which field to store in the "value". Using the dot ``.`` character you can go deeper inside the JSON object. To look inside an array, use square brackets `[]`. Keys with special characters should be written inside double quotes ``"``. Keep in mind that when written inside ``hyfiles``, it'll already be between double quotes, so you'll have to escape them with the backslash character ``\\``. Examples: ``env.production[0].field`` ``production.keys[1].x5c[0][1][0]."with space"[3]`` Attributes: extract: A string defining the location of the field that needs to be extracted. For now this is still quite primitive, and cannot access data from JSON arrays. """ def __init__(self, name: str, extract: str) -> None: """Initializes the Json Plugin. Creates the Json Plugin and extracts the specified field. Args: name: A string with the name of the Plugin. extract: A string with the location of the JSON field to extract. """ super().__init__( name=name, function=self.extract_json_field, flags=self.NEEDS_RESPONSE, ) self.extract = extract
[docs] def extract_json_field( self, response: requests.models.Response ) -> Optional[str]: """Extracts the JSON field from the text. Given the JSON body as a string, extract the field and store it in the Plugin's "value" attribute. Args: text: A string with the JSON body. Returns: A string with the result of extraction. If no such field is found None will be returned. """ data = json.loads(response.text) json_filter = parse_json_filter(self.extract) is_valid = True temp = data for item in json_filter: if item.startswith("["): index = int(item.strip("[]")) if len(temp) > index: temp = temp[index] else: logging.warning( ( "JSON array index doesn't exist.", "Cannot extract plugin's value.", ) ) is_valid = False break else: if item in temp: temp = temp[item] else: logging.warning( ( "Key '%s' not found in the response body.", "Cannot extract plugin's value.", ), item, ) is_valid = False break if is_valid: self.value = str(temp) logging.debug("Json filter %s: %s", self.name, str(self.value)) else: self.value = None return self.value
[docs] def __str__(self) -> str: """Returns a string representation of the Plugin.""" return "Json:" + str(self.extract)
[docs]class Variable(Plugin): """Plugin to extract the value of a variable. For now only the username and password variables are supported. Use this when supplying credentials to the web application. """ def __init__(self, name: str) -> None: """Initializes the Variable Plugin. Creates a Variable object that will return the data from a previously defined variable. Args: name: The name of the variable. """ super().__init__( name=name, function=self.extract_variable, flags=self.NEEDS_USERDATA, )
[docs] def extract_variable(self, data: Dict[str, str] = None) -> Optional[str]: """Extracts the variable value. Given a dictionary with the predefined variables, return the value of the with the same name as the "name" attribute from this Plugin. Args: data: A dictionary with the predefined variables. Returns: A string with the value of the variable found. None if no such variable has been defined. """ if data and self.name in data: self.value = data[self.name] return self.value
[docs]class Command(Plugin): """Runs a shell command and extract the output.""" def __init__(self, name: str, command: str) -> None: """Initializes the Command Plugin. The specified command will be executed with os.popen() and the output with the stripped last newline, will be saved inside the value. Args: name: A unique identifier for the plugin. command: The command to be executed. """ self.command = command super().__init__( name=name, function=self.run_command, )
[docs] def run_command(self) -> Optional[str]: """Runs a command and returns its value. Given a dictionary with the predefined variables, return the value of the with the same name as the "name" attribute from this Plugin. Args: data: A dictionary with the predefined variables. Returns: A string with the value of the variable found. None if no such variable has been defined. """ self.value = os.popen(self.command).read().strip() return self.value
[docs]class Prompt(Plugin): """Plugin to ask the user for an input. Use this plugin when the value cannot be known in advance, for example when asking for multi-factor authentication code that is going to be sent over SMS. """ def __init__(self, name: str) -> None: """Initializes the Prompt Plugin. Creates a Prompt Plugin which will ask the user's input to get the Plugin's value. Args: name: A string containing the prompt asking the user for input. """ super().__init__(name=name, function=self.get_user_prompt)
[docs] def get_user_prompt(self) -> str: """Gets the value from user input. Creates a prompt asking the user for input and stores the value in the Plugin. """ self.value = None while not self.value: print("Please provide the input value") self.value = input(self.name + " = ") return self.value
class Alter(Plugin): """Plugin used to alter other plugin's value. If the value extracted from other plugins cannot be used in it's raw form and needs to be somehow processed, Alter plugin can be used to do that. Initialize it with the original plugin and a function which will process the string and return the modified value. Attributes: alter_function: A function which will be given the plugin's value. It should return a string with the processed value. """ def __init__( self, plugin: Plugin, alter_function: Callable[[str], Optional[str]] ) -> None: """Initializes the Alter Plugin. Given the original plugin, and a function to alter the data, initialize the object, and get the modified value. Args: plugin: The original Plugin where the value is to be found. alter_function: The Function with instructions on how to alter the value. """ self.alter_function = alter_function super().__init__( name=plugin.name, value=plugin.value, # Get plugin's value from userdata since it has # already been extracted from the original plugin. flags=self.NEEDS_USERDATA, function=self.process_value, ) def process_value(self, data: Dict[str, str] = None) -> Optional[str]: """Process the original plugin's value. Get the value of the original's plugin from userdata, and give it to ``alter_function``. Return the processed value and store it in self.value. Args: data: A dictionary with the userdata where the original plugin's value should be found. Returns: A string with the processed value. """ if data and self.name in data: self.value = self.alter_function(data[self.name]) return self.value @classmethod def prepend(cls, plugin: Plugin, string: str) -> "Alter": """Prepend a string to plugin's value.""" alter = cls(plugin=plugin, alter_function=lambda value: string + value) return alter @classmethod def append(cls, plugin: Plugin, string: str) -> "Alter": """Append a string after the plugin's value""" alter = cls(plugin=plugin, alter_function=lambda value: value + string) return alter