From 59eeb1321fbad41dce7c971778e37ac1943fda3b Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Wed, 5 Aug 2020 20:39:28 -0500 Subject: Redo package to have a flat package structure By having a flat package structure like this, it'll avoid exposing packages and things we don't want to be exposed. --- config_file/__init__.py | 16 +- config_file/abstract_parser.py | 117 ++++++ config_file/base_parser.py | 139 +++++++ config_file/config_file.py | 6 +- config_file/config_file_path.py | 21 +- config_file/default.py | 14 - config_file/exceptions.py | 2 +- config_file/ini_parser.py | 13 + config_file/ini_parser_compat.py | 42 ++ config_file/json_parser.py | 15 + config_file/nested_lookup.py | 402 +++++++++++++++++++ config_file/nested_lookup/__init__.py | 19 - config_file/nested_lookup/lookup_api.py | 214 ---------- config_file/nested_lookup/nested_lookup.py | 178 --------- config_file/parse_value.py | 218 ++++++++++ config_file/parsers/__init__.py | 0 config_file/parsers/abstract_parser.py | 117 ------ config_file/parsers/base_parser.py | 98 ----- config_file/parsers/ini_parser.py | 13 - config_file/parsers/ini_parser_compat.py | 42 -- config_file/parsers/json_parser.py | 15 - config_file/parsers/parse_value.py | 218 ---------- config_file/parsers/toml_parser.py | 28 -- config_file/parsers/yaml_parser.py | 28 -- config_file/toml_parser.py | 28 ++ config_file/utils.py | 16 + config_file/yaml_parser.py | 27 ++ tests/test_config_file.py | 98 ++--- tests/test_ini_parser.py | 4 +- tests/test_json_parser.py | 2 +- tests/test_lookup_api.py | 622 ----------------------------- tests/test_nested_lookup.py | 617 +++++++++++++++++++++++++++- tests/test_parse_value.py | 2 +- 33 files changed, 1677 insertions(+), 1714 deletions(-) create mode 100644 config_file/abstract_parser.py create mode 100644 config_file/base_parser.py delete mode 100644 config_file/default.py create mode 100644 config_file/ini_parser.py create mode 100644 config_file/ini_parser_compat.py create mode 100644 config_file/json_parser.py create mode 100644 config_file/nested_lookup.py delete mode 100644 config_file/nested_lookup/__init__.py delete mode 100644 config_file/nested_lookup/lookup_api.py delete mode 100644 config_file/nested_lookup/nested_lookup.py create mode 100644 config_file/parse_value.py delete mode 100644 config_file/parsers/__init__.py delete mode 100644 config_file/parsers/abstract_parser.py delete mode 100644 config_file/parsers/base_parser.py delete mode 100644 config_file/parsers/ini_parser.py delete mode 100644 config_file/parsers/ini_parser_compat.py delete mode 100644 config_file/parsers/json_parser.py delete mode 100644 config_file/parsers/parse_value.py delete mode 100644 config_file/parsers/toml_parser.py delete mode 100644 config_file/parsers/yaml_parser.py create mode 100644 config_file/toml_parser.py create mode 100644 config_file/yaml_parser.py delete mode 100644 tests/test_lookup_api.py diff --git a/config_file/__init__.py b/config_file/__init__.py index 3825140..d78baaf 100644 --- a/config_file/__init__.py +++ b/config_file/__init__.py @@ -1,21 +1,9 @@ -from config_file.config_file import ConfigFile -from config_file.exceptions import ( - ConfigFileError, - MissingDependencyError, - MissingKeyError, - ParsingError, - UnrecognizedFileError, -) -from config_file.parsers.abstract_parser import AbstractParser +from .abstract_parser import AbstractParser +from .config_file import ConfigFile __version__ = "0.10.0" __all__ = [ "ConfigFile", "AbstractParser", - "ConfigFileError", - "ParsingError", - "UnrecognizedFileError", - "MissingDependencyError", - "MissingKeyError", ] diff --git a/config_file/abstract_parser.py b/config_file/abstract_parser.py new file mode 100644 index 0000000..ed9430b --- /dev/null +++ b/config_file/abstract_parser.py @@ -0,0 +1,117 @@ +from abc import ABC, abstractmethod +from typing import Any + + +class AbstractParser(ABC): + """Abstract Parser is the API contract for all parsers. + + Every parser in ConfigFile implements this parser, and + any user created parsers must implement this parser in + order for this package to be able to use it. + + Every key is specified in a dot (.) syntax + e.g. retrieving.some.deeply.nested.value + + If there is no dot in the key, it is assumed we are + performing an action on a top level key, which could + be an entire section. + + Args: + file_contents: The stringified version of a file. + + Attributes: + file_contents: The stringified version of a file. + """ + + @abstractmethod + def __init__(self, file_contents: str) -> None: + self.file_contents = file_contents + + @abstractmethod + def reset_internal_contents(self, file_contents: str) -> None: + """Reset the state of this parser to a new file. + + When restoring to the original file in ConfigFile, + we need a way to reset the internal contents to this + new file. + + While this may be as simple as setting the file_contents + attribute value to the passed in file_contents argument, + it likely would not be. This is because the initial + passed in file contents would likely be parsed into some + special form for the parser to work with more easily. + + Args: + file_contents: The file contents of the new file. + """ + raise NotImplementedError + + @abstractmethod + def get(self, key: str) -> Any: + """Retrieve the value of a key from the working file. + + Args: + key: The key to retrieve. + + Returns: + Whatever value the given key happens to contain. + + Raises: + MissingKeyError: If the key to retrieve does not exist. + """ + raise NotImplementedError + + @abstractmethod + def set(self, key: str, value: Any) -> None: + """Set the value of a key in the working file. + + If the key does not exist, it should be created. + + Args: + key: The key to set. + value: The value to set the key to. + """ + raise NotImplementedError + + @abstractmethod + def delete(self, key: str) -> None: + """Deletes a key from the working file. + + Args: + key: The key we would like to remove. + + Raises: + MissingKeyError: If the key to remove does not exist. + """ + raise NotImplementedError + + @abstractmethod + def stringify(self) -> str: + """Stringify the working file. + + Since the ConfigFile does not write out to disk + after every operation, this method is what is + used as the output for saving the file again. + This parser should never write out to the disk. + + Returns: + The file we are working with as a string. + """ + raise NotImplementedError + + @abstractmethod + def has(self, key: str) -> bool: + """Check whether the given key is in the parsed working file. + + This should not behave like a wild card, as in check + if the key exists anywhere in the working file. Instead, + if a key is given without a dot, it should be assumed we + are checking the top level. + + Args: + key: The key to search for. + + Returns: + True if the working file has the key. False otherwise. + """ + raise NotImplementedError diff --git a/config_file/base_parser.py b/config_file/base_parser.py new file mode 100644 index 0000000..a987a55 --- /dev/null +++ b/config_file/base_parser.py @@ -0,0 +1,139 @@ +from types import ModuleType +from typing import Any, Type + +from config_file.abstract_parser import AbstractParser +from config_file.exceptions import MissingKeyError, ParsingError +from config_file.nested_lookup import get_occurrence_of_key +from config_file.utils import split_on_dot + + +class BaseParser(AbstractParser): + def __init__( + self, file_contents: str, module: ModuleType, decode_error: Type[Exception] + ): + """The BaseParser implements the AbstractParser for us by giving it any module. + + As long as the module has a `loads` and `dumps` method, the BaseParser can use + it. Every file is then represented and worked with as a dictionary. + + Args: + file_contents: The contents of the file to parse. + + module: The module to use to parse the files. + Must have a `loads` and `dumps` method. + + decode_error: The error the module raises when the + file cannot be decoded. + + Raises: + ParsingError: If the decode_error is raised. + """ + super().__init__(file_contents) + self.__module = module + self.__decode_error = decode_error + self.parsed_content = self.parse_file_contents(module, decode_error) + + def parse_file_contents(self, module: ModuleType, decode_error: Exception) -> dict: + """Parse the file contents by running the `loads` method on the module. + + Args: + module: The module to use to parse the file. + decode_error: The error that is raised from the module if the file + cannot be decoded. + + Raises: + ParsingError: If the decode_error is raised while running `loads`. + + Returns: + The parsed file as a dictionary. + """ + try: + return module.loads(self.file_contents) + except decode_error as error: + raise ParsingError(error) + + def reset_internal_contents(self, file_contents: str) -> None: + """Reset the file contents and parsed contents of the parser. + + Args: + file_contents: The new file contents. + + Raises: + ParsingError: If we're unable to parse the new file contents. + """ + self.file_contents = file_contents + self.parsed_content = self.parse_file_contents( + self.__module, self.__decode_error + ) + + def get(self, search_key: str) -> Any: + key_causing_error = None + + try: + if "." not in search_key: + key_causing_error = search_key + return self.parsed_content[search_key] + else: + split_keys = split_on_dot(search_key) + + content_reference = self.parsed_content + for key in split_keys: + key_causing_error = key + content_reference = content_reference[key] + + except (KeyError, TypeError): + raise KeyError( + f"cannot `get` {search_key} because " + f"{key_causing_error} is not subscriptable." + ) + + return content_reference + + def set(self, key: str, value: Any) -> None: + if "." not in key: + self.parsed_content[key] = value + return + + keys = split_on_dot(key) + content_reference = self.parsed_content + for index, key in enumerate(keys): + if index == len(keys) - 1: + content_reference[key] = value + return + + try: + content_reference = content_reference[key] + except KeyError: + content_reference[key] = {} + content_reference = content_reference[key] + + def delete(self, key: str) -> None: + try: + if "." not in key: + del self.parsed_content[key] + else: + indexes = split_on_dot(key) + indexes_length = len(indexes) + + content_reference = self.parsed_content + for index, key in enumerate(indexes): + if index == indexes_length - 1: + del content_reference[key] + break + + content_reference = content_reference[key] + except (KeyError, TypeError): + raise MissingKeyError(f"The specified key '{key}' to delete was not found.") + + def stringify(self) -> str: + return self.__module.dumps(self.parsed_content) + + def has(self, search_key: str, wild: bool = False) -> bool: + if wild: + return get_occurrence_of_key(self.parsed_content, key=search_key) > 0 + + try: + self.get(search_key) + return True + except (KeyError, ParsingError): + return False diff --git a/config_file/config_file.py b/config_file/config_file.py index 3e8a806..d047560 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -3,11 +3,11 @@ from pathlib import Path from shutil import copyfile from typing import Any, Optional, Type, Union +from config_file.abstract_parser import AbstractParser from config_file.config_file_path import ConfigFilePath -from config_file.default import Default from config_file.exceptions import ParsingError -from config_file.parsers.abstract_parser import AbstractParser -from config_file.parsers.parse_value import parse_value +from config_file.parse_value import parse_value +from config_file.utils import Default class ConfigFile: diff --git a/config_file/config_file_path.py b/config_file/config_file_path.py index a40cc40..9f2b87d 100644 --- a/config_file/config_file_path.py +++ b/config_file/config_file_path.py @@ -3,12 +3,13 @@ from pathlib import Path as _Path_ from pathlib import _posix_flavour, _windows_flavour from typing import Type -from .exceptions import UnrecognizedFileError -from .parsers.abstract_parser import AbstractParser -from .parsers.ini_parser import IniParser -from .parsers.json_parser import JsonParser -from .parsers.toml_parser import TomlParser -from .parsers.yaml_parser import YamlParser +from config_file.abstract_parser import AbstractParser +from config_file.ini_parser import IniParser +from config_file.json_parser import JsonParser +from config_file.toml_parser import TomlParser +from config_file.yaml_parser import YamlParser + +from .exceptions import UnsupportedFileTypeError from .utils import split_on_dot @@ -20,7 +21,7 @@ class ConfigFilePath(_Path_): that should be use for the given file type. See https://codereview.stackexchange.com/questions/162426/subclassing-pathlib-path - for most details on subclassing the pathlib.Path object. + for more details on subclassing the pathlib.Path object. """ _flavour = _windows_flavour if os.name == "nt" else _posix_flavour @@ -57,14 +58,14 @@ class ConfigFilePath(_Path_): """Determine what parser should be used for this file. Raises: - UnrecognizedFileError: If the extension of the file + UnsupportedFileTypeError: If the extension of the file is not recognized. Returns: The instantiated parser that should be used for this file. """ if self.extension == "": - raise UnrecognizedFileError( + raise UnsupportedFileTypeError( "Tried to determine a parser to use, but the file at " f"{self} does not have an extension." ) @@ -78,7 +79,7 @@ class ConfigFilePath(_Path_): elif self.extension == "toml": return TomlParser(self.contents) else: - raise UnrecognizedFileError( + raise UnsupportedFileTypeError( f"File path at `{self}` contains an unrecognized file type." ) diff --git a/config_file/default.py b/config_file/default.py deleted file mode 100644 index 529021c..0000000 --- a/config_file/default.py +++ /dev/null @@ -1,14 +0,0 @@ -class Default: - """ - Default is used for the `default` value in ConfigFile's `get`. - - Previously, the default value for `get` was None. However, then - the user cannot have `get` return a default value of None. So this - class is used instead to get around that limitation. - """ - - def __init__(self, value): - self.value = value - - def __repr__(self): - return "Default Value: {} ({})".format(self.value, type(self.value)) diff --git a/config_file/exceptions.py b/config_file/exceptions.py index d1b466b..22ee70c 100644 --- a/config_file/exceptions.py +++ b/config_file/exceptions.py @@ -9,7 +9,7 @@ class MissingDependencyError(ConfigFileError): """ -class UnrecognizedFileError(ConfigFileError): +class UnsupportedFileTypeError(ConfigFileError): """ A missing file extension or a file type not supported. """ diff --git a/config_file/ini_parser.py b/config_file/ini_parser.py new file mode 100644 index 0000000..72974f1 --- /dev/null +++ b/config_file/ini_parser.py @@ -0,0 +1,13 @@ +import configparser + +from .base_parser import BaseParser +from .ini_parser_compat import IniParserCompatibility + + +class IniParser(BaseParser): + def __init__(self, file_contents: str): + super().__init__( + file_contents=file_contents, + module=IniParserCompatibility, + decode_error=configparser.Error, + ) diff --git a/config_file/ini_parser_compat.py b/config_file/ini_parser_compat.py new file mode 100644 index 0000000..9dfea2e --- /dev/null +++ b/config_file/ini_parser_compat.py @@ -0,0 +1,42 @@ +import configparser +from io import StringIO + + +class IniParserCompatibility: + """ + Provides compatibility with the json, yaml, + and toml API so we can use the BaseParser + easily, while still being able to leverage + the built-in configparser to parse the file. + + It is used as the `module` parameter + for BaseParser, which requires us to have a + `dumps` and `loads` method. + """ + + @staticmethod + def loads(file_contents: str) -> dict: + parser = configparser.ConfigParser() + parser.read_string(file_contents) + + return IniParserCompatibility.__create_configparser_dict(parser) + + @staticmethod + def dumps(file_contents: dict) -> str: + parser = configparser.ConfigParser() + buffer = StringIO() + + parser.read_dict(file_contents) + parser.write(buffer) + + return buffer.getvalue() + + @staticmethod + def __create_configparser_dict(parser: configparser.ConfigParser) -> dict: + result = {} + items = dict(parser.items()) + + for section in parser.sections(): + result.update({str(section): dict(items[section])}) + + return result diff --git a/config_file/json_parser.py b/config_file/json_parser.py new file mode 100644 index 0000000..5314cd7 --- /dev/null +++ b/config_file/json_parser.py @@ -0,0 +1,15 @@ +import json + +from .base_parser import BaseParser + + +class JsonParser(BaseParser): + def __init__(self, file_contents: str): + super().__init__( + file_contents=file_contents, + module=json, + decode_error=json.decoder.JSONDecodeError, + ) + + def stringify(self) -> str: + return json.dumps(self.parsed_content, indent=4) diff --git a/config_file/nested_lookup.py b/config_file/nested_lookup.py new file mode 100644 index 0000000..40da123 --- /dev/null +++ b/config_file/nested_lookup.py @@ -0,0 +1,402 @@ +import copy +import warnings +from collections import defaultdict + +__all__ = [ + "get_all_keys", + "get_occurrences_and_values", + "get_occurrence_of_value", + "get_occurrence_of_key", + "nested_update", + "nested_delete", + "nested_alter", + "nested_lookup", +] + + +values_list = [] + + +def nested_lookup(key, document, wild=False, with_keys=False): + """Lookup a key in a nested document, return a list of values""" + if with_keys: + d = defaultdict(list) + for k, v in _nested_lookup(key, document, wild=wild, with_keys=with_keys): + d[k].append(v) + return d + return list(_nested_lookup(key, document, wild=wild, with_keys=with_keys)) + + +def _is_case_insensitive_substring(a, b): + """return True if `a` is a case insensitive substring of `b`, else False""" + return str(a).lower() in str(b).lower() + + +def _nested_lookup(key, document, wild=False, with_keys=False): + """Lookup a key in a nested document, yield a value""" + if isinstance(document, list): + for d in document: + for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): + yield result + + if isinstance(document, dict): + for k, v in document.items(): + if key == k or (wild and _is_case_insensitive_substring(key, k)): + if with_keys: + yield k, v + else: + yield v + if isinstance(v, dict): + for result in _nested_lookup(key, v, wild=wild, with_keys=with_keys): + yield result + elif isinstance(v, list): + for d in v: + for result in _nested_lookup( + key, d, wild=wild, with_keys=with_keys + ): + yield result + + +def get_all_keys(dictionary): + """ + Method to get all keys from a nested dictionary as a List + Args: + dictionary: Nested dictionary + Returns: + List of keys in the dictionary + """ + result_list = [] + + def recrusion(document): + if isinstance(document, list): + for list_items in document: + recrusion(document=list_items) + elif isinstance(document, dict): + for key, value in document.items(): + result_list.append(key) + recrusion(document=value) + return + + recrusion(document=dictionary) + return result_list + + +def get_occurrence_of_key(dictionary, key): + """ + Method to get occurrence of a key in a nested dictionary + + Args: + dictionary: Nested dictionary + key: Key to search for the occurrences + Return: + Number of occurrence (Integer) + """ + return _get_occurrence(dictionary=dictionary, item="key", keyword=key) + + +def get_occurrences_and_values(items, value): + """ + Method to get occurrence of a value in a nested list of dictionary + + Args: + items: list of dictionary: Nested dictionary + value: Value to search for the occurrences + + Return: + Dict where the key is the value arg and his value is a new + dict with occurrences and values + """ + occurrences = {} + occurrence = 0 + value_list = [] + + for item in items: + occurrence_result, values = _get_occurrence_with_values( + dictionary=item, item="value", keyword=value + ) + occurrence = occurrence + occurrence_result + if occurrence_result: + value_list.extend(values) + + occurrences[value] = {"occurrences": occurrence, "values": value_list} + + return occurrences + + +def _get_occurrence_with_values(dictionary, item, keyword): + occurrence = [0] + + result_recursion = _recursion(dictionary, item, keyword, occurrence, True) + + global values_list + values_list = [] + + return occurrence[0], result_recursion + + +def get_occurrence_of_value(dictionary, value): + """ + Method to get occurrence of a value in a nested dictionary + + Args: + dictionary: Nested dictionary + value: Value to search for the occurrences + Return: + Number of occurrence (Integer) + """ + return _get_occurrence(dictionary=dictionary, item="value", keyword=value) + + +def _recursion(dictionary, item, keyword, occurrence, with_values=False): + + global values_list + + if item == "key": + if dictionary.get(keyword) is not None: + occurrence[0] += 1 + elif keyword in list(dictionary.values()): + occurrence[0] += list(dictionary.values()).count(keyword) + if with_values: + values_list.append(dictionary) + for key, value in dictionary.items(): + if isinstance(value, dict): + _recursion(value, item, keyword, occurrence, with_values) + elif isinstance(value, list): + for list_items in value: + if hasattr(list_items, "items"): + _recursion(list_items, item, keyword, occurrence, with_values) + elif list_items == keyword: + occurrence[0] += 1 if item == "value" else 0 + + if values_list: + return values_list + + +def _get_occurrence(dictionary, item, keyword): + """ + Method to get occurrence of a key or value in a nested dictionary + + Args: + dictionary: Nested dictionary + item: Mostly (key or value) + keyword: key word to find occurrence + Return: + Number of occurrence of the given keyword in the dict + """ + occurrence = [0] + _recursion(dictionary, item, keyword, occurrence) + + global values_list + values_list = [] + + return occurrence[0] + + +def nested_delete(document, key, in_place=False): + if not in_place: + document = copy.deepcopy(document) + return _nested_delete(document=document, key=key) + + +def _nested_delete(document, key): + """ + Method to delete a key->value pair from a nested document + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to delete + Return: + Returns a document that includes everything but the given key + """ + if isinstance(document, list): + for list_items in document: + _nested_delete(document=list_items, key=key) + elif isinstance(document, dict): + if document.get(key): + del document[key] + for dict_key, dict_value in document.items(): + _nested_delete(document=dict_value, key=key) + return document + + +def nested_update(document, key, value, in_place=False, treat_as_element=True): + """ + Method to update a key->value pair in a nested document + + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + + key: Key to update the value + + value: Value to set + + in_place (bool): + True: modify the dict in place; + False: create a deep copy of the dict and modify it + Defaults to False + + treat_as_element (bool): + True: if a list is provided as "value", the function trys + to match the list elements to the occurences of the key. + If the key occures more often than the provided list has + elements, the first element gets recycled. + False: the provided list is treated as one scalar value and + will be set as value to every key that matches. + Defaults to True (because of backwards portability of the package). + + Return: + Returns a document that has updated key, value pair. + """ + + # check if a list or scalar value is provided and create a list + # from the scalar value + # check the length of the list and provide it to _nested_update + if not treat_as_element and not isinstance(value, list): + raise Exception("The value must be a list when treat_as_element is False.") + elif treat_as_element: + value = [value] + + if not in_place: + document = copy.deepcopy(document) + return _nested_update(document=document, key=key, value=value) + + +def _nested_update(document, key, value): + """ + Method to update a key->value pair in a nested document. + If the number of passed values is less than the number of key matches + when scanning for updates, use last value as default. + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key (str): Key to update the value + value (list): value(s) which should be used for replacement purpouse + Return: + Returns a document that has updated key, value pair. + """ + if isinstance(document, list): + for list_items in document: + _nested_update(document=list_items, key=key, value=value) + elif isinstance(document, dict): + for dict_key, dict_value in document.items(): + if dict_key == key: + document[key] = value[0] + if len(value) > 1: + value.pop(0) + _nested_update(document=dict_value, key=key, value=value) + return document + + +def nested_alter( + document, + key, + callback_function=None, + function_parameters=None, + conversion_function=None, + wild_alter=False, + in_place=False, +): + """ + Method to alter all values of the occurences of the key "key". + The provided callback_function is used to alter the scalar values + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to update the value + callback_function :A callback function which alters a scalar value + HINT: You should be aware that not every element might be of + the same type, please check this in your function! + function_parameters (list): + If the callback_function has additional input arguments except + the scalar value, please specify those in this list. + conversion_function: A conversion function like str() which should be + applied to every found value before it is passed to the + "callback_function" + wild_alter: Find matching elements via wild-match by the given keys + and alter those. + HINT: Keep in mind that the wild-match might return unexpected types! + in_place (bool): + True: modify the dict in place; + False: create a deep copy of the dict and modify it + Defaults to False + Return: + Returns a document that has updated key, value pair. + """ + # check if a list or scalar value is provided and create a list from + # the scalar value + # check the length of the list and provide it to _nested_update + if isinstance(key, list): + key_len = len(key) + else: + key = [key] + key_len = len(key) + + if not in_place: + document = copy.deepcopy(document) + return _nested_alter( + document=document, + keys=key, + callback_function=callback_function, + function_parameters=function_parameters, + conversion_function=conversion_function, + wild_alter=wild_alter, + in_place=in_place, + key_len=key_len, + ) + + +def _call_callback( + value_list, callback_function, function_parameters, conversion_function +): + """ + internal helper to call the callback function + """ + return_list = [] + # loop over all values + for value in value_list: + # apply the conversion function + if conversion_function is not None: + value = conversion_function(value) + # if functions arguments are present, expand the list to variables + # via the magic operator * + if function_parameters: + trans_val = callback_function(value, *function_parameters) + else: + trans_val = callback_function(value) + # append the transformed element to the list + return_list.append(trans_val) + return return_list + + +def _nested_alter( + document, + keys, + callback_function, + function_parameters, + conversion_function, + wild_alter, + in_place, + key_len, +): + + # return data if no callback_function is provided + if callback_function is None: + warnings.warn("Please provide a callback_function to nested_alter().") + return document + + # iterate over all given keys in the list + for key in keys: + # try to find the key: + findings = nested_lookup(key, document, with_keys=True, wild=wild_alter) + for k, v in findings.items(): + trans_val = _call_callback( + v, callback_function, function_parameters, conversion_function + ) + # use the transformed value and apply the update to the key + # (dont treat the lists as elements here) + document = nested_update( + document, k, trans_val, in_place=in_place, treat_as_element=False + ) + + return document diff --git a/config_file/nested_lookup/__init__.py b/config_file/nested_lookup/__init__.py deleted file mode 100644 index 4493bdf..0000000 --- a/config_file/nested_lookup/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from .lookup_api import nested_alter, nested_delete, nested_update -from .nested_lookup import ( - get_all_keys, - get_occurrence_of_key, - get_occurrence_of_value, - get_occurrences_and_values, - nested_lookup, -) - -__all__ = [ - "get_all_keys", - "get_occurrences_and_values", - "get_occurrence_of_value", - "get_occurrence_of_key", - "nested_update", - "nested_delete", - "nested_alter", - "nested_lookup", -] diff --git a/config_file/nested_lookup/lookup_api.py b/config_file/nested_lookup/lookup_api.py deleted file mode 100644 index 886e8bf..0000000 --- a/config_file/nested_lookup/lookup_api.py +++ /dev/null @@ -1,214 +0,0 @@ -import copy -import warnings - -from .nested_lookup import nested_lookup - - -def nested_delete(document, key, in_place=False): - if not in_place: - document = copy.deepcopy(document) - return _nested_delete(document=document, key=key) - - -def _nested_delete(document, key): - """ - Method to delete a key->value pair from a nested document - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key: Key to delete - Return: - Returns a document that includes everything but the given key - """ - if isinstance(document, list): - for list_items in document: - _nested_delete(document=list_items, key=key) - elif isinstance(document, dict): - if document.get(key): - del document[key] - for dict_key, dict_value in document.items(): - _nested_delete(document=dict_value, key=key) - return document - - -def nested_update(document, key, value, in_place=False, treat_as_element=True): - """ - Method to update a key->value pair in a nested document - - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - - key: Key to update the value - - value: Value to set - - in_place (bool): - True: modify the dict in place; - False: create a deep copy of the dict and modify it - Defaults to False - - treat_as_element (bool): - True: if a list is provided as "value", the function trys - to match the list elements to the occurences of the key. - If the key occures more often than the provided list has - elements, the first element gets recycled. - False: the provided list is treated as one scalar value and - will be set as value to every key that matches. - Defaults to True (because of backwards portability of the package). - - Return: - Returns a document that has updated key, value pair. - """ - - # check if a list or scalar value is provided and create a list - # from the scalar value - # check the length of the list and provide it to _nested_update - if not treat_as_element and not isinstance(value, list): - raise Exception("The value must be a list when treat_as_element is False.") - elif treat_as_element: - value = [value] - - if not in_place: - document = copy.deepcopy(document) - return _nested_update(document=document, key=key, value=value) - - -def _nested_update(document, key, value): - """ - Method to update a key->value pair in a nested document. - If the number of passed values is less than the number of key matches - when scanning for updates, use last value as default. - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key (str): Key to update the value - value (list): value(s) which should be used for replacement purpouse - Return: - Returns a document that has updated key, value pair. - """ - if isinstance(document, list): - for list_items in document: - _nested_update(document=list_items, key=key, value=value) - elif isinstance(document, dict): - for dict_key, dict_value in document.items(): - if dict_key == key: - document[key] = value[0] - if len(value) > 1: - value.pop(0) - _nested_update(document=dict_value, key=key, value=value) - return document - - -def nested_alter( - document, - key, - callback_function=None, - function_parameters=None, - conversion_function=None, - wild_alter=False, - in_place=False, -): - """ - Method to alter all values of the occurences of the key "key". - The provided callback_function is used to alter the scalar values - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key: Key to update the value - callback_function :A callback function which alters a scalar value - HINT: You should be aware that not every element might be of - the same type, please check this in your function! - function_parameters (list): - If the callback_function has additional input arguments except - the scalar value, please specify those in this list. - conversion_function: A conversion function like str() which should be - applied to every found value before it is passed to the - "callback_function" - wild_alter: Find matching elements via wild-match by the given keys - and alter those. - HINT: Keep in mind that the wild-match might return unexpected types! - in_place (bool): - True: modify the dict in place; - False: create a deep copy of the dict and modify it - Defaults to False - Return: - Returns a document that has updated key, value pair. - """ - # check if a list or scalar value is provided and create a list from - # the scalar value - # check the length of the list and provide it to _nested_update - if isinstance(key, list): - key_len = len(key) - else: - key = [key] - key_len = len(key) - - if not in_place: - document = copy.deepcopy(document) - return _nested_alter( - document=document, - keys=key, - callback_function=callback_function, - function_parameters=function_parameters, - conversion_function=conversion_function, - wild_alter=wild_alter, - in_place=in_place, - key_len=key_len, - ) - - -def _call_callback( - value_list, callback_function, function_parameters, conversion_function -): - """ - internal helper to call the callback function - """ - return_list = [] - # loop over all values - for value in value_list: - # apply the conversion function - if conversion_function is not None: - value = conversion_function(value) - # if functions arguments are present, expand the list to variables - # via the magic operator * - if function_parameters: - trans_val = callback_function(value, *function_parameters) - else: - trans_val = callback_function(value) - # append the transformed element to the list - return_list.append(trans_val) - return return_list - - -def _nested_alter( - document, - keys, - callback_function, - function_parameters, - conversion_function, - wild_alter, - in_place, - key_len, -): - - # return data if no callback_function is provided - if callback_function is None: - warnings.warn("Please provide a callback_function to nested_alter().") - return document - - # iterate over all given keys in the list - for key in keys: - # try to find the key: - findings = nested_lookup(key, document, with_keys=True, wild=wild_alter) - for k, v in findings.items(): - trans_val = _call_callback( - v, callback_function, function_parameters, conversion_function - ) - # use the transformed value and apply the update to the key - # (dont treat the lists as elements here) - document = nested_update( - document, k, trans_val, in_place=in_place, treat_as_element=False - ) - - return document diff --git a/config_file/nested_lookup/nested_lookup.py b/config_file/nested_lookup/nested_lookup.py deleted file mode 100644 index 6f2e7f2..0000000 --- a/config_file/nested_lookup/nested_lookup.py +++ /dev/null @@ -1,178 +0,0 @@ -from collections import defaultdict - -values_list = [] - - -def nested_lookup(key, document, wild=False, with_keys=False): - """Lookup a key in a nested document, return a list of values""" - if with_keys: - d = defaultdict(list) - for k, v in _nested_lookup(key, document, wild=wild, with_keys=with_keys): - d[k].append(v) - return d - return list(_nested_lookup(key, document, wild=wild, with_keys=with_keys)) - - -def _is_case_insensitive_substring(a, b): - """return True if `a` is a case insensitive substring of `b`, else False""" - return str(a).lower() in str(b).lower() - - -def _nested_lookup(key, document, wild=False, with_keys=False): - """Lookup a key in a nested document, yield a value""" - if isinstance(document, list): - for d in document: - for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): - yield result - - if isinstance(document, dict): - for k, v in document.items(): - if key == k or (wild and _is_case_insensitive_substring(key, k)): - if with_keys: - yield k, v - else: - yield v - if isinstance(v, dict): - for result in _nested_lookup(key, v, wild=wild, with_keys=with_keys): - yield result - elif isinstance(v, list): - for d in v: - for result in _nested_lookup( - key, d, wild=wild, with_keys=with_keys - ): - yield result - - -def get_all_keys(dictionary): - """ - Method to get all keys from a nested dictionary as a List - Args: - dictionary: Nested dictionary - Returns: - List of keys in the dictionary - """ - result_list = [] - - def recrusion(document): - if isinstance(document, list): - for list_items in document: - recrusion(document=list_items) - elif isinstance(document, dict): - for key, value in document.items(): - result_list.append(key) - recrusion(document=value) - return - - recrusion(document=dictionary) - return result_list - - -def get_occurrence_of_key(dictionary, key): - """ - Method to get occurrence of a key in a nested dictionary - - Args: - dictionary: Nested dictionary - key: Key to search for the occurrences - Return: - Number of occurrence (Integer) - """ - return _get_occurrence(dictionary=dictionary, item="key", keyword=key) - - -def get_occurrences_and_values(items, value): - """ - Method to get occurrence of a value in a nested list of dictionary - - Args: - items: list of dictionary: Nested dictionary - value: Value to search for the occurrences - - Return: - Dict where the key is the value arg and his value is a new - dict with occurrences and values - """ - occurrences = {} - occurrence = 0 - value_list = [] - - for item in items: - occurrence_result, values = _get_occurrence_with_values( - dictionary=item, item="value", keyword=value - ) - occurrence = occurrence + occurrence_result - if occurrence_result: - value_list.extend(values) - - occurrences[value] = {"occurrences": occurrence, "values": value_list} - - return occurrences - - -def _get_occurrence_with_values(dictionary, item, keyword): - occurrence = [0] - - result_recursion = _recursion(dictionary, item, keyword, occurrence, True) - - global values_list - values_list = [] - - return occurrence[0], result_recursion - - -def get_occurrence_of_value(dictionary, value): - """ - Method to get occurrence of a value in a nested dictionary - - Args: - dictionary: Nested dictionary - value: Value to search for the occurrences - Return: - Number of occurrence (Integer) - """ - return _get_occurrence(dictionary=dictionary, item="value", keyword=value) - - -def _recursion(dictionary, item, keyword, occurrence, with_values=False): - - global values_list - - if item == "key": - if dictionary.get(keyword) is not None: - occurrence[0] += 1 - elif keyword in list(dictionary.values()): - occurrence[0] += list(dictionary.values()).count(keyword) - if with_values: - values_list.append(dictionary) - for key, value in dictionary.items(): - if isinstance(value, dict): - _recursion(value, item, keyword, occurrence, with_values) - elif isinstance(value, list): - for list_items in value: - if hasattr(list_items, "items"): - _recursion(list_items, item, keyword, occurrence, with_values) - elif list_items == keyword: - occurrence[0] += 1 if item == "value" else 0 - - if values_list: - return values_list - - -def _get_occurrence(dictionary, item, keyword): - """ - Method to get occurrence of a key or value in a nested dictionary - - Args: - dictionary: Nested dictionary - item: Mostly (key or value) - keyword: key word to find occurrence - Return: - Number of occurrence of the given keyword in the dict - """ - occurrence = [0] - _recursion(dictionary, item, keyword, occurrence) - - global values_list - values_list = [] - - return occurrence[0] diff --git a/config_file/parse_value.py b/config_file/parse_value.py new file mode 100644 index 0000000..12032d6 --- /dev/null +++ b/config_file/parse_value.py @@ -0,0 +1,218 @@ +"""This module allows you to parse a string into its native type.""" +import ast +import re +from distutils.util import strtobool +from typing import Any + + +def parse_value(value: Any) -> Any: + """Parse a value into native type. + + parse_value('5') -> 5 (int) + parse_value('-5') -> -5 (int) + + parse_value('5.52') -> 5.52 (float) + parse_value('-53.2') -> -53.2 (float) + + parse_value('blahfafda') -> 'blahfafda' (string) + + parse_value('true') -> True (bool) + parse_value('FALSE') -> False (bool) + + parse_value(" [ [ [ [] ]] ] ") -> [[[[]]]] (list) + + Args: + value: The value to parse. + + Returns: + The parsed value, if it could be parsed + as a dictionary, list, int, float, or bool. + Otherwise, it simply returns the original + passed in value. + """ + if isinstance(value, dict): + parsed = {} + + for item in value: + parsed[item] = parse_value(value[item]) + + return parsed + + if isinstance(value, list): + parsed = [] + + for item in value: + parsed.append(parse_value(item)) + + return parsed + + if can_be_parsed_as_int(value): + return int(value.strip()) if isinstance(value, str) else int(value) + + if can_be_parsed_as_float(value): + return float(value.strip()) if isinstance(value, str) else float(value) + + if can_be_parsed_as_bool(value): + return bool(value) if isinstance(value, bool) else bool(strtobool(value)) + + if can_be_parsed_as_dict(value): + return parse_value(ast.literal_eval(value.strip())) + + if can_be_parsed_as_list(value): + return parse_value(ast.literal_eval(value.strip())) + + return value + + +def can_be_parsed_as_int(value: Any) -> bool: + """Check whether a value can be parsed as a integer. + + can_be_parsed_as_int(5) -> True + can_be_parsed_as_int("5") -> True + can_be_parsed_as_int("-5") -> True + can_be_parsed_as_int(True) -> False + + Args: + value: The value to check if it can be parsed as an integer. + + Returns: + True if we can parse the value as a integer. False otherwise. + """ + if isinstance(value, int) and not isinstance(value, bool): + return True + + if not isinstance(value, str): + return False + + value = value.strip() + + if value.startswith("-"): + return value[1:].isdigit() + + return value.isdigit() + + +def can_be_parsed_as_float(value: Any) -> bool: + """Checks whether a value can be parsed as a float. + + can_be_parsed_as_float(45) -> False + can_be_parsed_as_float(1.1) -> True + can_be_parsed_as_float("1.1") -> True + can_be_parsed_as_float("-1.1") -> True + can_be_parsed_as_float(".1") -> True + + Args: + value: The value to be parsed. + + Returns: + True if the value can be parsed as a float. False otherwise. + """ + FLOAT_REGEX = r"^\d*\.\d+$" + + if isinstance(value, float): + return True + + if not isinstance(value, str): + return False + + value = value.strip() + + if value.startswith("-"): + value = value[1:] + + return bool(re.match(FLOAT_REGEX, value)) + + +def can_be_parsed_as_bool(value: Any) -> bool: + """Checks whether a value can be parsed as a boolean. + + can_be_parsed_as_bool(True) -> True + can_be_parsed_as_bool("true") -> True + can_be_parsed_as_bool("false") -> True + can_be_parsed_as_bool("TRUE") -> True + can_be_parsed_as_bool(0) -> False + + Args: + value: The value to be parsed. + + Returns: + True if the value can be parsed as a boolean. False otherwise. + """ + if isinstance(value, bool): + return True + + if not isinstance(value, str): + return False + + value = value.lower().strip() + + return value == "true" or value == "false" + + +def can_be_parsed_as_dict(value: Any) -> bool: + """Checks whether a value can be parsed as a dictionary. + + can_be_parsed_as_dict({}) -> True + can_be_parsed_as_dict("{}") -> True + can_be_parsed_as_dict(" { } ") -> True + can_be_parsed_as_dict("{ 'hello': 5 }") -> True + can_be_parsed_as_dict([]]) -> False + can_be_parsed_as_dict(" { invalid } ") -> False + + Args: + value: The value to be parsed. + + Returns: + True if the value can be parsed as a dictionary. False otherwise. + """ + if isinstance(value, dict): + return True + + if not isinstance(value, str): + return False + + value = value.strip() + + if value[0] == "{" and value[-1] == "}": + try: + ast.literal_eval(value) + return True + except Exception: + return False + + return False + + +def can_be_parsed_as_list(value: Any) -> bool: + """Checks whether a value can be parsed as a list. + + can_be_parsed_as_list([]) -> True + can_be_parsed_as_list([1, 2]) -> True + can_be_parsed_as_list("[]") -> True + can_be_parsed_as_list(" [] ") -> True + can_be_parsed_as_list("[1, 2]") -> True + can_be_parsed_as_list({}) -> False + can_be_parsed_as_list(" [ invalid ] ") -> False + + Args: + value: The value to be parsed. + + Returns: + True if the value can be parsed as a list. False otherwise. + """ + if isinstance(value, list): + return True + + if not isinstance(value, str): + return False + + value = value.strip() + + if value[0] == "[" and value[-1] == "]": + try: + ast.literal_eval(value) + return True + except Exception: + return False + + return False diff --git a/config_file/parsers/__init__.py b/config_file/parsers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/config_file/parsers/abstract_parser.py b/config_file/parsers/abstract_parser.py deleted file mode 100644 index ed9430b..0000000 --- a/config_file/parsers/abstract_parser.py +++ /dev/null @@ -1,117 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any - - -class AbstractParser(ABC): - """Abstract Parser is the API contract for all parsers. - - Every parser in ConfigFile implements this parser, and - any user created parsers must implement this parser in - order for this package to be able to use it. - - Every key is specified in a dot (.) syntax - e.g. retrieving.some.deeply.nested.value - - If there is no dot in the key, it is assumed we are - performing an action on a top level key, which could - be an entire section. - - Args: - file_contents: The stringified version of a file. - - Attributes: - file_contents: The stringified version of a file. - """ - - @abstractmethod - def __init__(self, file_contents: str) -> None: - self.file_contents = file_contents - - @abstractmethod - def reset_internal_contents(self, file_contents: str) -> None: - """Reset the state of this parser to a new file. - - When restoring to the original file in ConfigFile, - we need a way to reset the internal contents to this - new file. - - While this may be as simple as setting the file_contents - attribute value to the passed in file_contents argument, - it likely would not be. This is because the initial - passed in file contents would likely be parsed into some - special form for the parser to work with more easily. - - Args: - file_contents: The file contents of the new file. - """ - raise NotImplementedError - - @abstractmethod - def get(self, key: str) -> Any: - """Retrieve the value of a key from the working file. - - Args: - key: The key to retrieve. - - Returns: - Whatever value the given key happens to contain. - - Raises: - MissingKeyError: If the key to retrieve does not exist. - """ - raise NotImplementedError - - @abstractmethod - def set(self, key: str, value: Any) -> None: - """Set the value of a key in the working file. - - If the key does not exist, it should be created. - - Args: - key: The key to set. - value: The value to set the key to. - """ - raise NotImplementedError - - @abstractmethod - def delete(self, key: str) -> None: - """Deletes a key from the working file. - - Args: - key: The key we would like to remove. - - Raises: - MissingKeyError: If the key to remove does not exist. - """ - raise NotImplementedError - - @abstractmethod - def stringify(self) -> str: - """Stringify the working file. - - Since the ConfigFile does not write out to disk - after every operation, this method is what is - used as the output for saving the file again. - This parser should never write out to the disk. - - Returns: - The file we are working with as a string. - """ - raise NotImplementedError - - @abstractmethod - def has(self, key: str) -> bool: - """Check whether the given key is in the parsed working file. - - This should not behave like a wild card, as in check - if the key exists anywhere in the working file. Instead, - if a key is given without a dot, it should be assumed we - are checking the top level. - - Args: - key: The key to search for. - - Returns: - True if the working file has the key. False otherwise. - """ - raise NotImplementedError diff --git a/config_file/parsers/base_parser.py b/config_file/parsers/base_parser.py deleted file mode 100644 index c2a7d71..0000000 --- a/config_file/parsers/base_parser.py +++ /dev/null @@ -1,98 +0,0 @@ -from types import ModuleType -from typing import Any - -from config_file.exceptions import MissingKeyError, ParsingError -from config_file.nested_lookup import get_occurrence_of_key -from config_file.utils import split_on_dot - -from .abstract_parser import AbstractParser - - -class BaseParser(AbstractParser): - def __init__(self, file_contents: str, module: ModuleType, decode_error: Exception): - super().__init__(file_contents) - self.__module = module - self.__decode_error = decode_error - self.parsed_content = self.parse_file_contents(module, decode_error) - - def parse_file_contents(self, module, decode_error) -> Any: - try: - return module.loads(self.file_contents) - except decode_error as error: - raise ParsingError(error) - - def reset_internal_contents(self, file_contents: str) -> None: - self.file_contents = file_contents - self.parsed_content = self.parse_file_contents( - self.__module, self.__decode_error - ) - - def get(self, search_key: str) -> Any: - if "." not in search_key: - result = self.parsed_content[search_key] - return result - else: - split_keys = split_on_dot(search_key) - - result = self.parsed_content - key_causing_error = None - try: - for key in split_keys: - key_causing_error = key - result = result[key] - except (KeyError, TypeError): - raise ParsingError( - f"Cannot `get` {search_key} because " - f"{key_causing_error} is not subscriptable." - ) - - return result - - def set(self, key: str, value: Any) -> None: - if "." not in key: - self.parsed_content[key] = value - return - - keys = split_on_dot(key) - content_reference = self.parsed_content - for index, key in enumerate(keys): - if index == len(keys) - 1: - content_reference[key] = value - return - - try: - content_reference = content_reference[key] - except KeyError: - content_reference[key] = {} - content_reference = content_reference[key] - - def delete(self, key: str) -> None: - try: - if "." not in key: - del self.parsed_content[key] - else: - indexes = split_on_dot(key) - indexes_length = len(indexes) - - content_reference = self.parsed_content - for index, key in enumerate(indexes): - if index == indexes_length - 1: - del content_reference[key] - break - - content_reference = content_reference[key] - except (KeyError, TypeError): - raise MissingKeyError(f"The specified key '{key}' to delete was not found.") - - def stringify(self) -> str: - return self.__module.dumps(self.parsed_content) - - def has(self, search_key: str, wild: bool = False) -> bool: - if wild: - return get_occurrence_of_key(self.parsed_content, key=search_key) > 0 - - try: - self.get(search_key) - return True - except (KeyError, ParsingError): - return False diff --git a/config_file/parsers/ini_parser.py b/config_file/parsers/ini_parser.py deleted file mode 100644 index 72974f1..0000000 --- a/config_file/parsers/ini_parser.py +++ /dev/null @@ -1,13 +0,0 @@ -import configparser - -from .base_parser import BaseParser -from .ini_parser_compat import IniParserCompatibility - - -class IniParser(BaseParser): - def __init__(self, file_contents: str): - super().__init__( - file_contents=file_contents, - module=IniParserCompatibility, - decode_error=configparser.Error, - ) diff --git a/config_file/parsers/ini_parser_compat.py b/config_file/parsers/ini_parser_compat.py deleted file mode 100644 index 9dfea2e..0000000 --- a/config_file/parsers/ini_parser_compat.py +++ /dev/null @@ -1,42 +0,0 @@ -import configparser -from io import StringIO - - -class IniParserCompatibility: - """ - Provides compatibility with the json, yaml, - and toml API so we can use the BaseParser - easily, while still being able to leverage - the built-in configparser to parse the file. - - It is used as the `module` parameter - for BaseParser, which requires us to have a - `dumps` and `loads` method. - """ - - @staticmethod - def loads(file_contents: str) -> dict: - parser = configparser.ConfigParser() - parser.read_string(file_contents) - - return IniParserCompatibility.__create_configparser_dict(parser) - - @staticmethod - def dumps(file_contents: dict) -> str: - parser = configparser.ConfigParser() - buffer = StringIO() - - parser.read_dict(file_contents) - parser.write(buffer) - - return buffer.getvalue() - - @staticmethod - def __create_configparser_dict(parser: configparser.ConfigParser) -> dict: - result = {} - items = dict(parser.items()) - - for section in parser.sections(): - result.update({str(section): dict(items[section])}) - - return result diff --git a/config_file/parsers/json_parser.py b/config_file/parsers/json_parser.py deleted file mode 100644 index 5314cd7..0000000 --- a/config_file/parsers/json_parser.py +++ /dev/null @@ -1,15 +0,0 @@ -import json - -from .base_parser import BaseParser - - -class JsonParser(BaseParser): - def __init__(self, file_contents: str): - super().__init__( - file_contents=file_contents, - module=json, - decode_error=json.decoder.JSONDecodeError, - ) - - def stringify(self) -> str: - return json.dumps(self.parsed_content, indent=4) diff --git a/config_file/parsers/parse_value.py b/config_file/parsers/parse_value.py deleted file mode 100644 index 12032d6..0000000 --- a/config_file/parsers/parse_value.py +++ /dev/null @@ -1,218 +0,0 @@ -"""This module allows you to parse a string into its native type.""" -import ast -import re -from distutils.util import strtobool -from typing import Any - - -def parse_value(value: Any) -> Any: - """Parse a value into native type. - - parse_value('5') -> 5 (int) - parse_value('-5') -> -5 (int) - - parse_value('5.52') -> 5.52 (float) - parse_value('-53.2') -> -53.2 (float) - - parse_value('blahfafda') -> 'blahfafda' (string) - - parse_value('true') -> True (bool) - parse_value('FALSE') -> False (bool) - - parse_value(" [ [ [ [] ]] ] ") -> [[[[]]]] (list) - - Args: - value: The value to parse. - - Returns: - The parsed value, if it could be parsed - as a dictionary, list, int, float, or bool. - Otherwise, it simply returns the original - passed in value. - """ - if isinstance(value, dict): - parsed = {} - - for item in value: - parsed[item] = parse_value(value[item]) - - return parsed - - if isinstance(value, list): - parsed = [] - - for item in value: - parsed.append(parse_value(item)) - - return parsed - - if can_be_parsed_as_int(value): - return int(value.strip()) if isinstance(value, str) else int(value) - - if can_be_parsed_as_float(value): - return float(value.strip()) if isinstance(value, str) else float(value) - - if can_be_parsed_as_bool(value): - return bool(value) if isinstance(value, bool) else bool(strtobool(value)) - - if can_be_parsed_as_dict(value): - return parse_value(ast.literal_eval(value.strip())) - - if can_be_parsed_as_list(value): - return parse_value(ast.literal_eval(value.strip())) - - return value - - -def can_be_parsed_as_int(value: Any) -> bool: - """Check whether a value can be parsed as a integer. - - can_be_parsed_as_int(5) -> True - can_be_parsed_as_int("5") -> True - can_be_parsed_as_int("-5") -> True - can_be_parsed_as_int(True) -> False - - Args: - value: The value to check if it can be parsed as an integer. - - Returns: - True if we can parse the value as a integer. False otherwise. - """ - if isinstance(value, int) and not isinstance(value, bool): - return True - - if not isinstance(value, str): - return False - - value = value.strip() - - if value.startswith("-"): - return value[1:].isdigit() - - return value.isdigit() - - -def can_be_parsed_as_float(value: Any) -> bool: - """Checks whether a value can be parsed as a float. - - can_be_parsed_as_float(45) -> False - can_be_parsed_as_float(1.1) -> True - can_be_parsed_as_float("1.1") -> True - can_be_parsed_as_float("-1.1") -> True - can_be_parsed_as_float(".1") -> True - - Args: - value: The value to be parsed. - - Returns: - True if the value can be parsed as a float. False otherwise. - """ - FLOAT_REGEX = r"^\d*\.\d+$" - - if isinstance(value, float): - return True - - if not isinstance(value, str): - return False - - value = value.strip() - - if value.startswith("-"): - value = value[1:] - - return bool(re.match(FLOAT_REGEX, value)) - - -def can_be_parsed_as_bool(value: Any) -> bool: - """Checks whether a value can be parsed as a boolean. - - can_be_parsed_as_bool(True) -> True - can_be_parsed_as_bool("true") -> True - can_be_parsed_as_bool("false") -> True - can_be_parsed_as_bool("TRUE") -> True - can_be_parsed_as_bool(0) -> False - - Args: - value: The value to be parsed. - - Returns: - True if the value can be parsed as a boolean. False otherwise. - """ - if isinstance(value, bool): - return True - - if not isinstance(value, str): - return False - - value = value.lower().strip() - - return value == "true" or value == "false" - - -def can_be_parsed_as_dict(value: Any) -> bool: - """Checks whether a value can be parsed as a dictionary. - - can_be_parsed_as_dict({}) -> True - can_be_parsed_as_dict("{}") -> True - can_be_parsed_as_dict(" { } ") -> True - can_be_parsed_as_dict("{ 'hello': 5 }") -> True - can_be_parsed_as_dict([]]) -> False - can_be_parsed_as_dict(" { invalid } ") -> False - - Args: - value: The value to be parsed. - - Returns: - True if the value can be parsed as a dictionary. False otherwise. - """ - if isinstance(value, dict): - return True - - if not isinstance(value, str): - return False - - value = value.strip() - - if value[0] == "{" and value[-1] == "}": - try: - ast.literal_eval(value) - return True - except Exception: - return False - - return False - - -def can_be_parsed_as_list(value: Any) -> bool: - """Checks whether a value can be parsed as a list. - - can_be_parsed_as_list([]) -> True - can_be_parsed_as_list([1, 2]) -> True - can_be_parsed_as_list("[]") -> True - can_be_parsed_as_list(" [] ") -> True - can_be_parsed_as_list("[1, 2]") -> True - can_be_parsed_as_list({}) -> False - can_be_parsed_as_list(" [ invalid ] ") -> False - - Args: - value: The value to be parsed. - - Returns: - True if the value can be parsed as a list. False otherwise. - """ - if isinstance(value, list): - return True - - if not isinstance(value, str): - return False - - value = value.strip() - - if value[0] == "[" and value[-1] == "]": - try: - ast.literal_eval(value) - return True - except Exception: - return False - - return False diff --git a/config_file/parsers/toml_parser.py b/config_file/parsers/toml_parser.py deleted file mode 100644 index 1c7f8a7..0000000 --- a/config_file/parsers/toml_parser.py +++ /dev/null @@ -1,28 +0,0 @@ -try: - import tomlkit - - TOML_AVAILABLE = True -except ImportError: - TOML_AVAILABLE = False - -from config_file.exceptions import MissingDependencyError - -from .base_parser import BaseParser - - -class TomlParser(BaseParser): - def __init__(self, file_contents: str): - if not TOML_AVAILABLE: - raise MissingDependencyError( - "It doesn't appear `tomlkit` is installed, but a toml " - "file was attempted to be used. Install the `toml` " - "extra first with `pip install config-file[toml]`." - ) - - super().__init__( - file_contents=file_contents, - module=tomlkit, - # tomlkit doesn't expose it's exceptions, but - # it's parsing error is a subclass of ValueError - decode_error=ValueError, - ) diff --git a/config_file/parsers/yaml_parser.py b/config_file/parsers/yaml_parser.py deleted file mode 100644 index babe873..0000000 --- a/config_file/parsers/yaml_parser.py +++ /dev/null @@ -1,28 +0,0 @@ -try: - from ruamel.yaml import YAML, YAMLError, round_trip_dump, round_trip_load - - YAML_AVAILABLE = True -except ImportError: - YAML_AVAILABLE = False - -from config_file.exceptions import MissingDependencyError - -from .base_parser import BaseParser - - -class YamlParser(BaseParser): - def __init__(self, file_contents: str): - if not YAML_AVAILABLE: - raise MissingDependencyError( - "It doesn't appear `ruamel.yaml` is installed, but a yaml " - "file was attempted to be used. Install the `yaml` " - "extra first with `pip install config-file[yaml]`." - ) - - yaml = YAML(typ="safe") - yaml.loads = round_trip_load - yaml.dumps = round_trip_dump - - super().__init__( - file_contents=file_contents, module=yaml, decode_error=YAMLError - ) diff --git a/config_file/toml_parser.py b/config_file/toml_parser.py new file mode 100644 index 0000000..1c7f8a7 --- /dev/null +++ b/config_file/toml_parser.py @@ -0,0 +1,28 @@ +try: + import tomlkit + + TOML_AVAILABLE = True +except ImportError: + TOML_AVAILABLE = False + +from config_file.exceptions import MissingDependencyError + +from .base_parser import BaseParser + + +class TomlParser(BaseParser): + def __init__(self, file_contents: str): + if not TOML_AVAILABLE: + raise MissingDependencyError( + "It doesn't appear `tomlkit` is installed, but a toml " + "file was attempted to be used. Install the `toml` " + "extra first with `pip install config-file[toml]`." + ) + + super().__init__( + file_contents=file_contents, + module=tomlkit, + # tomlkit doesn't expose it's exceptions, but + # it's parsing error is a subclass of ValueError + decode_error=ValueError, + ) diff --git a/config_file/utils.py b/config_file/utils.py index c6c3d24..3554c0e 100644 --- a/config_file/utils.py +++ b/config_file/utils.py @@ -20,3 +20,19 @@ def split_on_dot(line: Union[str, Path], only_last_dot=False) -> List[str]: raise ValueError(f"The given string does not have a dot to split on: {line}") return line.rsplit(".", 1) if only_last_dot else line.split(".") + + +class Default: + """ + Default is used for the `default` value in ConfigFile's `get`. + + Previously, the default value for `get` was None. However, then + the user cannot have `get` return a default value of None. So this + class is used instead to get around that limitation. + """ + + def __init__(self, value): + self.value = value + + def __repr__(self): + return "Default Value: {} ({})".format(self.value, type(self.value)) diff --git a/config_file/yaml_parser.py b/config_file/yaml_parser.py new file mode 100644 index 0000000..fb42e08 --- /dev/null +++ b/config_file/yaml_parser.py @@ -0,0 +1,27 @@ +try: + from ruamel.yaml import YAML, YAMLError, round_trip_dump, round_trip_load + + YAML_AVAILABLE = True +except ImportError: + YAML_AVAILABLE = False + +from .base_parser import BaseParser +from .exceptions import MissingDependencyError + + +class YamlParser(BaseParser): + def __init__(self, file_contents: str): + if not YAML_AVAILABLE: + raise MissingDependencyError( + "It doesn't appear `ruamel.yaml` is installed, but a yaml " + "file was attempted to be used. Install the `yaml` " + "extra first with `pip install config-file[yaml]`." + ) + + yaml = YAML(typ="safe") + yaml.loads = round_trip_load + yaml.dumps = round_trip_dump + + super().__init__( + file_contents=file_contents, module=yaml, decode_error=YAMLError + ) diff --git a/tests/test_config_file.py b/tests/test_config_file.py index 548951b..91cd0e6 100644 --- a/tests/test_config_file.py +++ b/tests/test_config_file.py @@ -1,11 +1,10 @@ -import json from pathlib import Path import pytest +from config_file.abstract_parser import AbstractParser from config_file.config_file import ConfigFile from config_file.exceptions import MissingKeyError -from config_file.parsers.abstract_parser import AbstractParser SUPPORTED_FILE_TYPES = ["ini", "json", "yaml", "toml"] @@ -46,9 +45,14 @@ def test_that_a_tidle_in_the_config_path_expands_to_the_absolute_path(): Path(config_path).expanduser().unlink() -@pytest.mark.skip(msg="needs fixing up") -@pytest.mark.parametrize("path", ["", "invalid", "invalid.conf"]) -def test_invalid_path_raises_value_error(path): +@pytest.mark.parametrize("path", ["invalid", "invalid.conf"]) +def test_invalid_path_raises_file_not_found_error(path): + with pytest.raises(FileNotFoundError): + ConfigFile(path) + + +@pytest.mark.parametrize("path", [""]) +def test_directory_raises_value_error(path): with pytest.raises(ValueError): ConfigFile(path) @@ -99,75 +103,21 @@ def test_that_config_file_can_find_sections_and_keys( assert config.has(key) == expected_result -@pytest.mark.parametrize( - "section_key, value, parse_type, return_type, default, file_name, file_contents", - [ - ( - "calendar.sunday_index", - 0, - True, - None, - False, - "config.ini", - "[calendar]\nsunday_index = 0\n\n", - ), - ( - "calendar.sunday_index", - "0", - False, - None, - False, - "config.ini", - "[calendar]\nsunday_index = 0\n\n", - ), - ( - "calendar.sunday_index", - 0, - False, - None, - False, - "config.json", - json.dumps({"calendar": {"sunday_index": 0}}), - ), - ( - "calendar.missing", - "my default value", - False, - None, - "my default value", - "config.json", - json.dumps({"calendar": 5}), - ), - ( - "calendar.sunday_index", - 0, - False, - int, - False, - "config.ini", - "[calendar]\nsunday_index = 0\n\n", - ), - ], -) -def test_that_config_file_can_get( - tmpdir, - section_key, - value, - return_type, - parse_type, - file_name, - file_contents, - default, -): - config_path = tmpdir / file_name - config_path.write_text(file_contents, encoding="utf-8") - config = ConfigFile(config_path) - - ret_val = config.get(section_key, parse_types=parse_type, default=default) - if return_type is not None: - assert return_type(ret_val) == value - else: - assert ret_val == value +# @pytest.mark.parametrize( +# "key, value", +# [ +# ("header_one.number_key", 0), +# ("header_two.list_key", [1, 2, 3]), +# ], +# ) +# def test_that_config_file_can_get_existent_keys(templated_config_file, key, value): +# config = templated_config_file() +# +# print(config.get(key)) +# print(type(config.get(key))) +# print(config._ConfigFile__parser) +# +# assert config.get(key) == value @pytest.mark.parametrize("key", ["header_one.number_key", "header_one"]) diff --git a/tests/test_ini_parser.py b/tests/test_ini_parser.py index 46966c9..24507b7 100644 --- a/tests/test_ini_parser.py +++ b/tests/test_ini_parser.py @@ -1,7 +1,7 @@ import pytest from config_file.exceptions import MissingKeyError, ParsingError -from config_file.parsers.ini_parser import IniParser +from config_file.ini_parser import IniParser @pytest.mark.parametrize("ini_file", ["blahblahblah", "{ 'key1': 5 }"]) @@ -39,7 +39,7 @@ floatkey = 5.3 "test_input", ["invalid.invalid_key"], ) def test_that_getting_unknown_sections_or_keys_throws_errors(test_input): - with pytest.raises(ParsingError): + with pytest.raises(KeyError): IniParser("").get(test_input) diff --git a/tests/test_json_parser.py b/tests/test_json_parser.py index f8200b0..54ea646 100644 --- a/tests/test_json_parser.py +++ b/tests/test_json_parser.py @@ -84,7 +84,7 @@ def test_that_json_parser_can_get_keys_without_parsing_types( @pytest.mark.parametrize( - "key, value", [("zip", "zipzip"), ("foo.bar.foobar", 5), ("foo.baz", "bar")] + "key, value", [("zip", "zipzip"), ("foo.bar.foobar", "5"), ("foo.baz", "bar")] ) def test_that_json_parser_can_set_keys(templated_parser, key, value): parser = templated_parser("json") diff --git a/tests/test_lookup_api.py b/tests/test_lookup_api.py deleted file mode 100644 index 8b8ca9e..0000000 --- a/tests/test_lookup_api.py +++ /dev/null @@ -1,622 +0,0 @@ -from unittest import TestCase - -from config_file.nested_lookup import ( - nested_alter, - nested_delete, - nested_lookup, - nested_update, -) - - -class BaseLookUpApi(TestCase): - def setUp(self): - self.sample_data1 = { - "build_version": { - "model_name": "MacBook Pro", - "build_version": { - "processor_name": "Intel Core i7", - "processor_speed": "2.7 GHz", - "core_details": { - "build_version": "4", - "l2_cache(per_core)": "256 KB", - }, - }, - "number_of_cores": "4", - "memory": "256 KB", - }, - "os_details": {"product_version": "10.13.6", "build_version": "17G65"}, - "name": "Test", - "date": "YYYY-MM-DD HH:MM:SS", - } - - self.sample_data2 = { - "hardware_details": { - "model_name": "MacBook Pro", - "processor_details": [ - {"processor_name": "Intel Core i7", "processor_speed": "2.7 GHz"}, - {"total_number_of_cores": "4", "l2_cache(per_core)": "256 KB"}, - ], - "total_number_of_cores": "5", - "memory": "16 GB", - } - } - - self.sample_data3 = { - "values": [ - { - "checks": [ - { - "monitoring_zones": [ - "mzdfw", - "mzfra", - "mzhkg", - "mziad", - "mzlon", - "mzord", - "mzsyd", - ] - } - ] - } - ] - } - - self.sample_data4 = { - "modelversion": "1.1.0", - "vorgangsID": "1", - "versorgungsvorschlagDatum": 1510558834978, - "eingangsdatum": 1510558834978, - "plz": 82269, - "vertragsteile": [ - { - "typ": "1", - "beitragsDaten": { - "endalter": 85, - "brutto": 58.76, - "netto": 58.76, - "zahlungsrhythmus": "MONATLICH", - "plz": 86899, - }, - "beginn": 1512082800000, - "lebenslang": "True", - "ueberschussverwendung": { - "ueberschussverwendung": "2", - "indexoption": "3", - }, - "deckung": [ - { - "typ": "2", - "art": "1", - "leistung": {"value": 7500242424.0, "einheit": "2"}, - "leistungsRhythmus": "1", - } - ], - "zuschlagNachlass": [], - }, - { - "typ": "1", - "beitragsDaten": { - "endalter": 85, - "brutto": 0.6, - "netto": 0.6, - "zahlungsrhythmus": "1", - }, - "zuschlagNachlass": [], - }, - ], - } - - -class TestNestedDelete(BaseLookUpApi): - def test_sample_data1(self): - result = { - "os_details": {"product_version": "10.13.6"}, - "name": "Test", - "date": "YYYY-MM-DD HH:MM:SS", - } - self.assertEqual(result, nested_delete(self.sample_data1, "build_version")) - - def test_sample_data2(self): - result = { - "hardware_details": { - "model_name": "MacBook Pro", - "total_number_of_cores": "5", - "memory": "16 GB", - } - } - self.assertEqual(result, nested_delete(self.sample_data2, "processor_details")) - - def test_sample_data3(self): - result = {"values": [{"checks": [{}]}]} - self.assertEqual(result, nested_delete(self.sample_data3, "monitoring_zones")) - - -class TestNestedUpdate(BaseLookUpApi): - def test_sample_data1(self): - result = { - "build_version": "Test1", - "os_details": {"product_version": "10.13.6", "build_version": "Test1"}, - "name": "Test", - "date": "YYYY-MM-DD HH:MM:SS", - } - self.assertEqual( - result, nested_update(self.sample_data1, "build_version", "Test1") - ) - - def test_sample_data1_list_input_treat_list_as_element_true(self): - result = { - "build_version": ["Test5", "Test6", "Test7"], - "os_details": { - "product_version": "10.13.6", - "build_version": ["Test5", "Test6", "Test7"], - }, - "name": "Test", - "date": "YYYY-MM-DD HH:MM:SS", - } - - self.assertEqual( - result, - nested_update( - self.sample_data1, - "build_version", - ["Test5", "Test6", "Test7"], - treat_as_element=True, - ), - ) - - def test_nested_update_in_place_false(self): - """ - ested_update should mutate and return a copy of the original document - """ - before_id = id(self.sample_data1) - result = nested_update( - self.sample_data1, "build_version", "Test2", in_place=False - ) - after_id = id(result) - # the object ids should _not_ match. - self.assertNotEqual(before_id, after_id) - - def test_nested_update_in_place_true(self): - """ - nested_update should mutate and return the original document - """ - before_id = id(self.sample_data1) - result = nested_update( - self.sample_data1, "build_version", "Test2", in_place=True - ) - after_id = id(result) - # the object ids should match. - self.assertEqual(before_id, after_id) - - def test_nested_update_in_place_true_list_input(self): - doc = self.sample_data4 - # get all instances of the given element - findings = nested_lookup("plz", doc, False, True) - # alter those instances - updated_findings = list() - for key, val in findings.items(): - for elem in val: - updated_findings.append(elem + 300) - # update those instances with the altered results - doc_updated = nested_update( - doc, "plz", updated_findings, treat_as_element=False - ) - elem1 = doc_updated["plz"] # 85269 - # 87199 - elem2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] - self.assertEqual(elem1, 82569) - self.assertEqual(elem2, 87199) - - def test_nested_update_in_place_false_list_input(self): - doc = self.sample_data4 - # get all instances of the given element - findings = nested_lookup("plz", doc, False, True) - # alter those instances - updated_findings = list() - for key, val in findings.items(): - for elem in val: - updated_findings.append(elem + 300) - # update those instances with the altered results - doc_updated = nested_update( - doc, "plz", updated_findings, in_place=False, treat_as_element=False - ) - elem1 = doc_updated["plz"] - elem2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] - self.assertEqual(elem1, 82569) - self.assertEqual(elem2, 87199) - - def test_nested_update_in_place_false_list_input_as_element_false(self): - doc = self.sample_data4 - # get all instances of the given element - list_input = [1, 2, 3, 4, 5] - # update those instances with the altered results - doc_updated = nested_update( - doc, "plz", list_input, in_place=False, treat_as_element=False - ) - elem1 = doc_updated["plz"] - elem2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] - # should not work without specifying "treat_list_as_element = True" - # in nested_update - self.assertNotEqual(elem1, list_input) - self.assertNotEqual(elem2, list_input) - - def test_nested_update_in_place_false_list_input_as_element_true(self): - doc = self.sample_data4 - # get all instances of the given element - list_input = [1, 2, 3, 4, 5] - # update those instances with the altered results - doc_updated = nested_update( - doc, "plz", list_input, in_place=False, treat_as_element=True - ) - elem1 = doc_updated["plz"] - elem2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] - # should not work without specifying "treat_list_as_element = True" - # in nested_update - self.assertEqual(elem1, list_input) - self.assertEqual(elem2, list_input) - - def test_nested_update_in_place_true_list_input_as_element_false(self): - doc = self.sample_data4 - # get all instances of the given element - list_input = [1, 2, 3, 4, 5] - # update those instances with the altered results - doc_updated = nested_update( - doc, "plz", list_input, in_place=True, treat_as_element=False - ) - elem1 = doc_updated["plz"] - elem2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] - # should not work without specifying "treat_list_as_element = True" - # in nested_update - self.assertNotEqual(elem1, list_input) - self.assertNotEqual(elem2, list_input) - - def test_nested_update_in_place_true_list_input_as_element_true(self): - doc = self.sample_data4 - # get all instances of the given element - list_input = [1, 2, 3, 4, 5] - # update those instances with the altered results - doc_updated = nested_update( - doc, "plz", list_input, in_place=True, treat_as_element=True - ) - elem1 = doc_updated["plz"] - elem2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] - # should not work without specifying "treat_list_as_element = True" - # in nested_update - self.assertEqual(elem1, list_input) - self.assertEqual(elem2, list_input) - - def test_nested_delete_in_place_false(self): - """ - nested_delete should mutate and return a copy of the original document - """ - before_id = id(self.sample_data1) - result = nested_delete(self.sample_data1, "build_version", in_place=False) - after_id = id(result) - # the object ids should _not_ match. - self.assertNotEqual(before_id, after_id) - - def test_nested_delete_in_place_true(self): - """nested_delete should mutate and return the original document""" - before_id = id(self.sample_data1) - result = nested_delete(self.sample_data1, "build_version", in_place=True) - after_id = id(result) - # the object ids should match. - self.assertEqual(before_id, after_id) - - def test_nested_update_taco_for_example(self): - document = [{"taco": 42}, {"salsa": [{"burrito": {"taco": 69}}]}] - - updated_document = nested_update( - document, "taco", [100, 200], treat_as_element=False - ) - - self.assertEqual(updated_document[0]["taco"], 100) - # The multi-update version only works for scalar input, - # if you need to adress a list of dicts, you have to - # manually iterate over those and pass them to nested_update - # one by one - self.assertEqual(updated_document[1]["salsa"][0]["burrito"]["taco"], 200) - - def test_nested_update_raise_error(self): - doc = self.sample_data4 - # get all instances of the given element - list_input = 1 - # update those instances with the altered results - self.assertRaises( - Exception, - nested_update, - doc, - "plz", - list_input, - in_place=True, - treat_as_element=False, - ) - - def test_sample_data2(self): - result = { - "hardware_details": { - "model_name": "MacBook Pro", - "processor_details": {"test_key1": "test_value1"}, - "total_number_of_cores": "5", - "memory": "16 GB", - } - } - self.assertEqual( - result, - nested_update( - self.sample_data2, "processor_details", {"test_key1": "test_value1"} - ), - ) - - def test_sample_data3(self): - result = {"values": [{"checks": {"key1": ["value1"], "key2": "value2"}}]} - self.assertEqual( - result, - nested_update( - self.sample_data3, "checks", {"key1": ["value1"], "key2": "value2"} - ), - ) - - def test_sample_data4(self): - result = { - "modelversion": {"key1": ["value1"], "key2": "value2"}, - "vorgangsID": "1", - "versorgungsvorschlagDatum": 1510558834978, - "eingangsdatum": 1510558834978, - "plz": 82269, - "vertragsteile": [ - { - "typ": "1", - "beitragsDaten": { - "endalter": 85, - "brutto": 58.76, - "netto": 58.76, - "zahlungsrhythmus": "MONATLICH", - "plz": 86899, - }, - "beginn": 1512082800000, - "lebenslang": "True", - "ueberschussverwendung": { - "ueberschussverwendung": "2", - "indexoption": "3", - }, - "deckung": [ - { - "typ": "2", - "art": "1", - "leistung": {"value": 7500242424.0, "einheit": "2"}, - "leistungsRhythmus": "1", - } - ], - "zuschlagNachlass": [], - }, - { - "typ": "1", - "beitragsDaten": { - "endalter": 85, - "brutto": 0.6, - "netto": 0.6, - "zahlungsrhythmus": "1", - }, - "zuschlagNachlass": [], - }, - ], - } - self.assertEqual( - result, - nested_update( - self.sample_data4, - "modelversion", - {"key1": ["value1"], "key2": "value2"}, - ), - ) - - -class TestNestedAlter(BaseLookUpApi): - def test_nested_alter_in_place_true(self): - - # callback functions - def callback(data): - return str(data) + "###" - - doc_updated = nested_alter( - self.sample_data4, "vorgangsID", callback, in_place=True - ) - - vorgangsid = doc_updated["vorgangsID"] - - self.assertEqual(vorgangsid, "1###") - - def test_nested_alter_in_place_false(self): - - # callback functions - def callback(data): - return str(data) + "###" - - doc_updated = nested_alter( - self.sample_data4, "vorgangsID", callback, in_place=False - ) - - vorgangsid = doc_updated["vorgangsID"] - - # should not work without specifying - # "treat_list_as_element = True" in nested_update - self.assertEqual(vorgangsid, "1###") - - def test_nested_alter_list_input_in_place_true(self): - - # callback functions - def callback(data): - return str(data) + "###" - - doc_updated = nested_alter( - self.sample_data4, ["plz", "vorgangsID"], callback, in_place=True - ) - - plz1 = doc_updated["plz"] - plz2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] - vorgangsid = doc_updated["vorgangsID"] - - # should not work without specifying - # "treat_list_as_element = True" in nested_update - self.assertEqual(plz1, "82269###") - self.assertEqual(plz2, "86899###") - self.assertEqual(vorgangsid, "1###") - - def test_nested_alter_list_input_with_args_in_place_true(self): - - # callback functions - def callback(data, str1, str2): - return str(data) + str1 + str2 - - doc_updated = nested_alter( - self.sample_data4, - ["plz", "vorgangsID"], - callback, - function_parameters=["abc", "def"], - in_place=True, - ) - - plz1 = doc_updated["plz"] - plz2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] - vorgangsid = doc_updated["vorgangsID"] - - # should not work without specifying - # "treat_list_as_element = True" in nested_update - self.assertEqual(plz1, "82269abcdef") - self.assertEqual(plz2, "86899abcdef") - self.assertEqual(vorgangsid, "1abcdef") - - def test_nested_alter_list_input_with_args_in_place_false(self): - - # callback functions - def callback(data, str1, str2): - return str(data) + str1 + str2 - - doc_updated = nested_alter( - self.sample_data4, - ["plz", "vorgangsID"], - callback, - function_parameters=["abc", "def"], - in_place=False, - ) - - plz1 = doc_updated["plz"] - plz2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] - vorgangsid = doc_updated["vorgangsID"] - - # should not work without specifying - # "treat_list_as_element = True" in nested_update - self.assertEqual(plz1, "82269abcdef") - self.assertEqual(plz2, "86899abcdef") - self.assertEqual(vorgangsid, "1abcdef") - - def test_nested_alter_list_input_in_place_false(self): - - # callback functions - def callback(data): - return str(data) + "###" - - doc_updated = nested_alter( - self.sample_data4, ["plz", "vorgangsID"], callback, in_place=False - ) - - plz1 = doc_updated["plz"] - plz2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] - vorgangsid = doc_updated["vorgangsID"] - - # should not work without specifying - # "treat_list_as_element = True" in nested_update - self.assertEqual(plz1, "82269###") - self.assertEqual(plz2, "86899###") - self.assertEqual(vorgangsid, "1###") - - def test_nested_alter_taco_for_example(self): - documents = [{"taco": 42}, {"salsa": [{"burrito": {"taco": 69}}]}] - - # write a callback function which processes a scalar value. - # Be aware about the possible types which can be passed to - # the callback functions. - # In this example we can be sure that only int will be passed, - # in production you should check the type because it could be - # anything. - def callback(data): - return data + 10 # (data/100*10) - - # The alter-version only works for scalar input (one dict), - # if you need to adress a list of dicts, you have to - # manually iterate over those and pass them to - # nested_update one by one - out = [] - for elem in documents: - altered_document = nested_alter(elem, "taco", callback) - out.append(altered_document) - - self.maxDiff = None - self.assertEqual(out[0]["taco"], 52) - self.assertEqual(out[1]["salsa"][0]["burrito"]["taco"], 79) - - def test_nested_alter_work_with_right_order(self): - document = {"taco": 42, "salsa": [{"burrito": {"key": 20}}], "key": 50} - - def callback(data): - return data + 100 - - altered_document = nested_alter(document, "key", callback, in_place=True) - - self.assertEqual(altered_document["salsa"][0]["burrito"]["key"], 120) - self.assertEqual(altered_document["key"], 150) - - def test_sample_data4(self): - - result = { - "modelversion": "1.1.0", - "vorgangsID": "1", - "versorgungsvorschlagDatum": 1510558834978, - "eingangsdatum": 1510558834978, - "plz": 82270, - "vertragsteile": [ - { - "typ": "1", - "beitragsDaten": { - "endalter": 85, - "brutto": 58.76, - "netto": 58.76, - "zahlungsrhythmus": "MONATLICH", - "plz": 86900, - }, - "beginn": 1512082800000, - "lebenslang": "True", - "ueberschussverwendung": { - "ueberschussverwendung": "2", - "indexoption": "3", - }, - "deckung": [ - { - "typ": "2", - "art": "1", - "leistung": {"value": 7500242424.0, "einheit": "2"}, - "leistungsRhythmus": "1", - } - ], - "zuschlagNachlass": [], - }, - { - "typ": "1", - "beitragsDaten": { - "endalter": 85, - "brutto": 0.6, - "netto": 0.6, - "zahlungsrhythmus": "1", - }, - "zuschlagNachlass": [], - }, - ], - } - - # add +1 to all plz - def callback(data): - return data + 1 - - self.maxDiff = None - self.assertEqual(result, nested_alter(self.sample_data4, "plz", callback)) diff --git a/tests/test_nested_lookup.py b/tests/test_nested_lookup.py index 591edda..1239fad 100644 --- a/tests/test_nested_lookup.py +++ b/tests/test_nested_lookup.py @@ -5,7 +5,10 @@ from config_file.nested_lookup import ( get_occurrence_of_key, get_occurrence_of_value, get_occurrences_and_values, + nested_alter, + nested_delete, nested_lookup, + nested_update, ) @@ -447,5 +450,615 @@ class TestGetOccurrence(TestCase): self.assertEqual(0, len(result[value]["values"])) -if __name__ == "__main__": - pass +class BaseLookUpApi(TestCase): + def setUp(self): + self.sample_data1 = { + "build_version": { + "model_name": "MacBook Pro", + "build_version": { + "processor_name": "Intel Core i7", + "processor_speed": "2.7 GHz", + "core_details": { + "build_version": "4", + "l2_cache(per_core)": "256 KB", + }, + }, + "number_of_cores": "4", + "memory": "256 KB", + }, + "os_details": {"product_version": "10.13.6", "build_version": "17G65"}, + "name": "Test", + "date": "YYYY-MM-DD HH:MM:SS", + } + + self.sample_data2 = { + "hardware_details": { + "model_name": "MacBook Pro", + "processor_details": [ + {"processor_name": "Intel Core i7", "processor_speed": "2.7 GHz"}, + {"total_number_of_cores": "4", "l2_cache(per_core)": "256 KB"}, + ], + "total_number_of_cores": "5", + "memory": "16 GB", + } + } + + self.sample_data3 = { + "values": [ + { + "checks": [ + { + "monitoring_zones": [ + "mzdfw", + "mzfra", + "mzhkg", + "mziad", + "mzlon", + "mzord", + "mzsyd", + ] + } + ] + } + ] + } + + self.sample_data4 = { + "modelversion": "1.1.0", + "vorgangsID": "1", + "versorgungsvorschlagDatum": 1510558834978, + "eingangsdatum": 1510558834978, + "plz": 82269, + "vertragsteile": [ + { + "typ": "1", + "beitragsDaten": { + "endalter": 85, + "brutto": 58.76, + "netto": 58.76, + "zahlungsrhythmus": "MONATLICH", + "plz": 86899, + }, + "beginn": 1512082800000, + "lebenslang": "True", + "ueberschussverwendung": { + "ueberschussverwendung": "2", + "indexoption": "3", + }, + "deckung": [ + { + "typ": "2", + "art": "1", + "leistung": {"value": 7500242424.0, "einheit": "2"}, + "leistungsRhythmus": "1", + } + ], + "zuschlagNachlass": [], + }, + { + "typ": "1", + "beitragsDaten": { + "endalter": 85, + "brutto": 0.6, + "netto": 0.6, + "zahlungsrhythmus": "1", + }, + "zuschlagNachlass": [], + }, + ], + } + + +class TestNestedDelete(BaseLookUpApi): + def test_sample_data1(self): + result = { + "os_details": {"product_version": "10.13.6"}, + "name": "Test", + "date": "YYYY-MM-DD HH:MM:SS", + } + self.assertEqual(result, nested_delete(self.sample_data1, "build_version")) + + def test_sample_data2(self): + result = { + "hardware_details": { + "model_name": "MacBook Pro", + "total_number_of_cores": "5", + "memory": "16 GB", + } + } + self.assertEqual(result, nested_delete(self.sample_data2, "processor_details")) + + def test_sample_data3(self): + result = {"values": [{"checks": [{}]}]} + self.assertEqual(result, nested_delete(self.sample_data3, "monitoring_zones")) + + +class TestNestedUpdate(BaseLookUpApi): + def test_sample_data1(self): + result = { + "build_version": "Test1", + "os_details": {"product_version": "10.13.6", "build_version": "Test1"}, + "name": "Test", + "date": "YYYY-MM-DD HH:MM:SS", + } + self.assertEqual( + result, nested_update(self.sample_data1, "build_version", "Test1") + ) + + def test_sample_data1_list_input_treat_list_as_element_true(self): + result = { + "build_version": ["Test5", "Test6", "Test7"], + "os_details": { + "product_version": "10.13.6", + "build_version": ["Test5", "Test6", "Test7"], + }, + "name": "Test", + "date": "YYYY-MM-DD HH:MM:SS", + } + + self.assertEqual( + result, + nested_update( + self.sample_data1, + "build_version", + ["Test5", "Test6", "Test7"], + treat_as_element=True, + ), + ) + + def test_nested_update_in_place_false(self): + """ + ested_update should mutate and return a copy of the original document + """ + before_id = id(self.sample_data1) + result = nested_update( + self.sample_data1, "build_version", "Test2", in_place=False + ) + after_id = id(result) + # the object ids should _not_ match. + self.assertNotEqual(before_id, after_id) + + def test_nested_update_in_place_true(self): + """ + nested_update should mutate and return the original document + """ + before_id = id(self.sample_data1) + result = nested_update( + self.sample_data1, "build_version", "Test2", in_place=True + ) + after_id = id(result) + # the object ids should match. + self.assertEqual(before_id, after_id) + + def test_nested_update_in_place_true_list_input(self): + doc = self.sample_data4 + # get all instances of the given element + findings = nested_lookup("plz", doc, False, True) + # alter those instances + updated_findings = list() + for key, val in findings.items(): + for elem in val: + updated_findings.append(elem + 300) + # update those instances with the altered results + doc_updated = nested_update( + doc, "plz", updated_findings, treat_as_element=False + ) + elem1 = doc_updated["plz"] # 85269 + # 87199 + elem2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] + self.assertEqual(elem1, 82569) + self.assertEqual(elem2, 87199) + + def test_nested_update_in_place_false_list_input(self): + doc = self.sample_data4 + # get all instances of the given element + findings = nested_lookup("plz", doc, False, True) + # alter those instances + updated_findings = list() + for key, val in findings.items(): + for elem in val: + updated_findings.append(elem + 300) + # update those instances with the altered results + doc_updated = nested_update( + doc, "plz", updated_findings, in_place=False, treat_as_element=False + ) + elem1 = doc_updated["plz"] + elem2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] + self.assertEqual(elem1, 82569) + self.assertEqual(elem2, 87199) + + def test_nested_update_in_place_false_list_input_as_element_false(self): + doc = self.sample_data4 + # get all instances of the given element + list_input = [1, 2, 3, 4, 5] + # update those instances with the altered results + doc_updated = nested_update( + doc, "plz", list_input, in_place=False, treat_as_element=False + ) + elem1 = doc_updated["plz"] + elem2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] + # should not work without specifying "treat_list_as_element = True" + # in nested_update + self.assertNotEqual(elem1, list_input) + self.assertNotEqual(elem2, list_input) + + def test_nested_update_in_place_false_list_input_as_element_true(self): + doc = self.sample_data4 + # get all instances of the given element + list_input = [1, 2, 3, 4, 5] + # update those instances with the altered results + doc_updated = nested_update( + doc, "plz", list_input, in_place=False, treat_as_element=True + ) + elem1 = doc_updated["plz"] + elem2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] + # should not work without specifying "treat_list_as_element = True" + # in nested_update + self.assertEqual(elem1, list_input) + self.assertEqual(elem2, list_input) + + def test_nested_update_in_place_true_list_input_as_element_false(self): + doc = self.sample_data4 + # get all instances of the given element + list_input = [1, 2, 3, 4, 5] + # update those instances with the altered results + doc_updated = nested_update( + doc, "plz", list_input, in_place=True, treat_as_element=False + ) + elem1 = doc_updated["plz"] + elem2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] + # should not work without specifying "treat_list_as_element = True" + # in nested_update + self.assertNotEqual(elem1, list_input) + self.assertNotEqual(elem2, list_input) + + def test_nested_update_in_place_true_list_input_as_element_true(self): + doc = self.sample_data4 + # get all instances of the given element + list_input = [1, 2, 3, 4, 5] + # update those instances with the altered results + doc_updated = nested_update( + doc, "plz", list_input, in_place=True, treat_as_element=True + ) + elem1 = doc_updated["plz"] + elem2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] + # should not work without specifying "treat_list_as_element = True" + # in nested_update + self.assertEqual(elem1, list_input) + self.assertEqual(elem2, list_input) + + def test_nested_delete_in_place_false(self): + """ + nested_delete should mutate and return a copy of the original document + """ + before_id = id(self.sample_data1) + result = nested_delete(self.sample_data1, "build_version", in_place=False) + after_id = id(result) + # the object ids should _not_ match. + self.assertNotEqual(before_id, after_id) + + def test_nested_delete_in_place_true(self): + """nested_delete should mutate and return the original document""" + before_id = id(self.sample_data1) + result = nested_delete(self.sample_data1, "build_version", in_place=True) + after_id = id(result) + # the object ids should match. + self.assertEqual(before_id, after_id) + + def test_nested_update_taco_for_example(self): + document = [{"taco": 42}, {"salsa": [{"burrito": {"taco": 69}}]}] + + updated_document = nested_update( + document, "taco", [100, 200], treat_as_element=False + ) + + self.assertEqual(updated_document[0]["taco"], 100) + # The multi-update version only works for scalar input, + # if you need to adress a list of dicts, you have to + # manually iterate over those and pass them to nested_update + # one by one + self.assertEqual(updated_document[1]["salsa"][0]["burrito"]["taco"], 200) + + def test_nested_update_raise_error(self): + doc = self.sample_data4 + # get all instances of the given element + list_input = 1 + # update those instances with the altered results + self.assertRaises( + Exception, + nested_update, + doc, + "plz", + list_input, + in_place=True, + treat_as_element=False, + ) + + def test_sample_data2(self): + result = { + "hardware_details": { + "model_name": "MacBook Pro", + "processor_details": {"test_key1": "test_value1"}, + "total_number_of_cores": "5", + "memory": "16 GB", + } + } + self.assertEqual( + result, + nested_update( + self.sample_data2, "processor_details", {"test_key1": "test_value1"} + ), + ) + + def test_sample_data3(self): + result = {"values": [{"checks": {"key1": ["value1"], "key2": "value2"}}]} + self.assertEqual( + result, + nested_update( + self.sample_data3, "checks", {"key1": ["value1"], "key2": "value2"} + ), + ) + + def test_sample_data4(self): + result = { + "modelversion": {"key1": ["value1"], "key2": "value2"}, + "vorgangsID": "1", + "versorgungsvorschlagDatum": 1510558834978, + "eingangsdatum": 1510558834978, + "plz": 82269, + "vertragsteile": [ + { + "typ": "1", + "beitragsDaten": { + "endalter": 85, + "brutto": 58.76, + "netto": 58.76, + "zahlungsrhythmus": "MONATLICH", + "plz": 86899, + }, + "beginn": 1512082800000, + "lebenslang": "True", + "ueberschussverwendung": { + "ueberschussverwendung": "2", + "indexoption": "3", + }, + "deckung": [ + { + "typ": "2", + "art": "1", + "leistung": {"value": 7500242424.0, "einheit": "2"}, + "leistungsRhythmus": "1", + } + ], + "zuschlagNachlass": [], + }, + { + "typ": "1", + "beitragsDaten": { + "endalter": 85, + "brutto": 0.6, + "netto": 0.6, + "zahlungsrhythmus": "1", + }, + "zuschlagNachlass": [], + }, + ], + } + self.assertEqual( + result, + nested_update( + self.sample_data4, + "modelversion", + {"key1": ["value1"], "key2": "value2"}, + ), + ) + + +class TestNestedAlter(BaseLookUpApi): + def test_nested_alter_in_place_true(self): + + # callback functions + def callback(data): + return str(data) + "###" + + doc_updated = nested_alter( + self.sample_data4, "vorgangsID", callback, in_place=True + ) + + vorgangsid = doc_updated["vorgangsID"] + + self.assertEqual(vorgangsid, "1###") + + def test_nested_alter_in_place_false(self): + + # callback functions + def callback(data): + return str(data) + "###" + + doc_updated = nested_alter( + self.sample_data4, "vorgangsID", callback, in_place=False + ) + + vorgangsid = doc_updated["vorgangsID"] + + # should not work without specifying + # "treat_list_as_element = True" in nested_update + self.assertEqual(vorgangsid, "1###") + + def test_nested_alter_list_input_in_place_true(self): + + # callback functions + def callback(data): + return str(data) + "###" + + doc_updated = nested_alter( + self.sample_data4, ["plz", "vorgangsID"], callback, in_place=True + ) + + plz1 = doc_updated["plz"] + plz2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] + vorgangsid = doc_updated["vorgangsID"] + + # should not work without specifying + # "treat_list_as_element = True" in nested_update + self.assertEqual(plz1, "82269###") + self.assertEqual(plz2, "86899###") + self.assertEqual(vorgangsid, "1###") + + def test_nested_alter_list_input_with_args_in_place_true(self): + + # callback functions + def callback(data, str1, str2): + return str(data) + str1 + str2 + + doc_updated = nested_alter( + self.sample_data4, + ["plz", "vorgangsID"], + callback, + function_parameters=["abc", "def"], + in_place=True, + ) + + plz1 = doc_updated["plz"] + plz2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] + vorgangsid = doc_updated["vorgangsID"] + + # should not work without specifying + # "treat_list_as_element = True" in nested_update + self.assertEqual(plz1, "82269abcdef") + self.assertEqual(plz2, "86899abcdef") + self.assertEqual(vorgangsid, "1abcdef") + + def test_nested_alter_list_input_with_args_in_place_false(self): + + # callback functions + def callback(data, str1, str2): + return str(data) + str1 + str2 + + doc_updated = nested_alter( + self.sample_data4, + ["plz", "vorgangsID"], + callback, + function_parameters=["abc", "def"], + in_place=False, + ) + + plz1 = doc_updated["plz"] + plz2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] + vorgangsid = doc_updated["vorgangsID"] + + # should not work without specifying + # "treat_list_as_element = True" in nested_update + self.assertEqual(plz1, "82269abcdef") + self.assertEqual(plz2, "86899abcdef") + self.assertEqual(vorgangsid, "1abcdef") + + def test_nested_alter_list_input_in_place_false(self): + + # callback functions + def callback(data): + return str(data) + "###" + + doc_updated = nested_alter( + self.sample_data4, ["plz", "vorgangsID"], callback, in_place=False + ) + + plz1 = doc_updated["plz"] + plz2 = doc_updated["vertragsteile"][0]["beitragsDaten"]["plz"] + vorgangsid = doc_updated["vorgangsID"] + + # should not work without specifying + # "treat_list_as_element = True" in nested_update + self.assertEqual(plz1, "82269###") + self.assertEqual(plz2, "86899###") + self.assertEqual(vorgangsid, "1###") + + def test_nested_alter_taco_for_example(self): + documents = [{"taco": 42}, {"salsa": [{"burrito": {"taco": 69}}]}] + + # write a callback function which processes a scalar value. + # Be aware about the possible types which can be passed to + # the callback functions. + # In this example we can be sure that only int will be passed, + # in production you should check the type because it could be + # anything. + def callback(data): + return data + 10 # (data/100*10) + + # The alter-version only works for scalar input (one dict), + # if you need to adress a list of dicts, you have to + # manually iterate over those and pass them to + # nested_update one by one + out = [] + for elem in documents: + altered_document = nested_alter(elem, "taco", callback) + out.append(altered_document) + + self.maxDiff = None + self.assertEqual(out[0]["taco"], 52) + self.assertEqual(out[1]["salsa"][0]["burrito"]["taco"], 79) + + def test_nested_alter_work_with_right_order(self): + document = {"taco": 42, "salsa": [{"burrito": {"key": 20}}], "key": 50} + + def callback(data): + return data + 100 + + altered_document = nested_alter(document, "key", callback, in_place=True) + + self.assertEqual(altered_document["salsa"][0]["burrito"]["key"], 120) + self.assertEqual(altered_document["key"], 150) + + def test_sample_data4(self): + + result = { + "modelversion": "1.1.0", + "vorgangsID": "1", + "versorgungsvorschlagDatum": 1510558834978, + "eingangsdatum": 1510558834978, + "plz": 82270, + "vertragsteile": [ + { + "typ": "1", + "beitragsDaten": { + "endalter": 85, + "brutto": 58.76, + "netto": 58.76, + "zahlungsrhythmus": "MONATLICH", + "plz": 86900, + }, + "beginn": 1512082800000, + "lebenslang": "True", + "ueberschussverwendung": { + "ueberschussverwendung": "2", + "indexoption": "3", + }, + "deckung": [ + { + "typ": "2", + "art": "1", + "leistung": {"value": 7500242424.0, "einheit": "2"}, + "leistungsRhythmus": "1", + } + ], + "zuschlagNachlass": [], + }, + { + "typ": "1", + "beitragsDaten": { + "endalter": 85, + "brutto": 0.6, + "netto": 0.6, + "zahlungsrhythmus": "1", + }, + "zuschlagNachlass": [], + }, + ], + } + + # add +1 to all plz + def callback(data): + return data + 1 + + self.maxDiff = None + self.assertEqual(result, nested_alter(self.sample_data4, "plz", callback)) diff --git a/tests/test_parse_value.py b/tests/test_parse_value.py index 853537c..976c613 100644 --- a/tests/test_parse_value.py +++ b/tests/test_parse_value.py @@ -2,7 +2,7 @@ from datetime import datetime import pytest -from config_file.parsers.parse_value import ( +from config_file.parse_value import ( can_be_parsed_as_bool, can_be_parsed_as_dict, can_be_parsed_as_float, -- cgit v1.2.3