From aa44fe7a60a4f3326cc47877b3e246b56d46d6ac Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 20 Jun 2020 13:41:35 -0500 Subject: Add python 3.5 support, ConfigFileError, and more type hints Also, get_all, set_all, and delete_all in JsonParser have been renamed to just all. --- config_file/__init__.py | 4 +++- config_file/config_file.py | 18 +++++++++--------- config_file/exceptions.py | 6 ++++++ config_file/parsers/base_parser.py | 3 --- config_file/parsers/ini_parser.py | 7 ++++--- config_file/parsers/json_parser.py | 21 +++++++++++---------- config_file/parsers/toml_parser.py | 0 config_file/parsers/yaml_parser.py | 0 8 files changed, 33 insertions(+), 26 deletions(-) create mode 100644 config_file/exceptions.py create mode 100644 config_file/parsers/toml_parser.py create mode 100644 config_file/parsers/yaml_parser.py (limited to 'config_file') diff --git a/config_file/__init__.py b/config_file/__init__.py index 3bae28a..f37019a 100644 --- a/config_file/__init__.py +++ b/config_file/__init__.py @@ -1,10 +1,12 @@ from config_file.config_file import ConfigFile -from config_file.parsers.base_parser import BaseParser, ParsingError +from config_file.parsers.base_parser import BaseParser +from config_file.exceptions import ParsingError, ConfigFileError __version__ = "0.10.0" __all__ = [ "ConfigFile", "ParsingError", + "ConfigFileError", "BaseParser", ] diff --git a/config_file/config_file.py b/config_file/config_file.py index 91202da..805fc49 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -1,15 +1,17 @@ import inspect from pathlib import Path, PurePath from shutil import copyfile +from typing import Type, Union -from config_file.parsers.base_parser import BaseParser, ParsingError +from config_file.parsers.base_parser import BaseParser from config_file.parsers.ini_parser import IniParser from config_file.parsers.json_parser import JsonParser from config_file.utils import split_on_dot +from config_file.exceptions import ParsingError class ConfigFile: - def __init__(self, file_path, parser: BaseParser = None): + def __init__(self, file_path: Type[Union[str, PurePath]], parser: Type[BaseParser] = None) -> None: """ Saves the config file path and expands it if needed, reads in the file contents, and determines what parser should be used for @@ -17,7 +19,6 @@ class ConfigFile: :param file_path: The path to your configuration file. :param parser: A custom parser you'd like used for your config file. - It must be a concrete implementation of BaseParser. :raises ValueError: If the specified file path does not have an extension that is supported or it is a directory. @@ -27,7 +28,7 @@ class ConfigFile: self.parser = self.__determine_parser(file_path, parser) @staticmethod - def __create_config_path(file_path, original: bool = False) -> str: + def __create_config_path(file_path: Type[Union[str, PurePath]], original: bool = False) -> str: if isinstance(file_path, PurePath): file_path = str(file_path) @@ -40,11 +41,11 @@ class ConfigFile: file_path = ".".join(file_parts) if Path(file_path).is_dir(): - raise ValueError(f"The specified config file ({file_path}) is a directory.") + raise ValueError("The specified config file ({}) is a directory.".format(file_path)) return file_path - def __determine_parser(self, file_path, parser: BaseParser): + def __determine_parser(self, file_path: Type[Union[str, PurePath]], parser: Type[BaseParser]) -> BaseParser: if isinstance(file_path, PurePath): file_path = str(file_path) @@ -58,8 +59,7 @@ class ConfigFile: return JsonParser(self.contents) else: raise ValueError( - f"File path contains an unsupported or " - f"unrecognized file type: {file_path}" + "File path contains an unsupported or unrecognized file type: {}".format(file_path) ) def __read_config_file(self): @@ -134,7 +134,7 @@ class ConfigFile: if not Path(original_file_path).exists(): raise OSError( - f"The {original_file_path} file to restore to does not exist." + "The {} file to restore to does not exist.".format(original_file_path) ) Path(self.path).expanduser().unlink() diff --git a/config_file/exceptions.py b/config_file/exceptions.py new file mode 100644 index 0000000..cd7e236 --- /dev/null +++ b/config_file/exceptions.py @@ -0,0 +1,6 @@ +class ConfigFileError(Exception): + """A generic error from the package.""" + +class ParsingError(ConfigFileError): + """Unable to parse the configuration file.""" + \ No newline at end of file diff --git a/config_file/parsers/base_parser.py b/config_file/parsers/base_parser.py index 42f93b1..5fc47c2 100644 --- a/config_file/parsers/base_parser.py +++ b/config_file/parsers/base_parser.py @@ -74,6 +74,3 @@ class BaseParser(ABC): """ raise NotImplementedError - -class ParsingError(Exception): - """Unable to parse the configuration file""" diff --git a/config_file/parsers/ini_parser.py b/config_file/parsers/ini_parser.py index 81f2d1b..31b0dcf 100644 --- a/config_file/parsers/ini_parser.py +++ b/config_file/parsers/ini_parser.py @@ -1,9 +1,10 @@ import configparser from io import StringIO -from config_file.parsers.base_parser import BaseParser, ParsingError +from config_file.parsers.base_parser import BaseParser from config_file.parsers.parse_value import parse_value from config_file.utils import split_on_dot +from config_file.exceptions import ParsingError class IniParser(BaseParser): @@ -95,7 +96,7 @@ class IniParser(BaseParser): if "." not in section_key: if not self.parsed_content.has_section(section_key): raise ValueError( - f"Cannot delete {section_key} because it is not in the config file." + "Cannot delete {} because it is not in the config file.".format(section_key) ) self.parsed_content.remove_section(section_key) @@ -104,7 +105,7 @@ class IniParser(BaseParser): section, key = split_on_dot(section_key, only_last_dot=True) if key not in self.parsed_content[section]: raise ValueError( - f"Cannot delete {section}.{key} because {key} is not in {section}." + "Cannot delete {section}.{key} because {key} is not in {section}.".format(section = section, key = key) ) self.parsed_content.remove_option(section, key) diff --git a/config_file/parsers/json_parser.py b/config_file/parsers/json_parser.py index 20eb085..4d93ca3 100644 --- a/config_file/parsers/json_parser.py +++ b/config_file/parsers/json_parser.py @@ -7,9 +7,10 @@ from nested_lookup import ( nested_update, ) -from config_file.parsers.base_parser import BaseParser, ParsingError +from config_file.parsers.base_parser import BaseParser from config_file.parsers.parse_value import parse_value from config_file.utils import split_on_dot +from config_file.exceptions import ParsingError class JsonParser(BaseParser): @@ -34,7 +35,7 @@ class JsonParser(BaseParser): >>> { "key2": "foo", "key1": "bar" } parser.get('key1.key2') >>> "foo" - parser.get('key1', get_all=True) + parser.get('key1', all=True) >>> [{ "key2": "foo", "key1": "bar" }, "bar"] """ @@ -48,7 +49,7 @@ class JsonParser(BaseParser): except json.decoder.JSONDecodeError as error: raise ParsingError(error) - def get(self, key: str, parse_types: bool = False, get_all: bool = False): + def get(self, key: str, parse_types: bool = False, all: bool = False): """ Retrieve values using a dot syntax. @@ -56,14 +57,14 @@ class JsonParser(BaseParser): :param parse_types: Whether you'd like the type parsed into its native one. - :param get_all: Specify whether you'd like to recursively receive + :param all: Specify whether you'd like to recursively receive all values that match the given key. Returned as a list. e.g. 'foo' would retrieve all values that have the key 'foo', anywhere in the json. :return: the value of the given key """ - if get_all: + if all: result = nested_lookup(key, self.parsed_content) elif "." in key: split_keys = split_on_dot(key) @@ -77,15 +78,15 @@ class JsonParser(BaseParser): result = result[key] except TypeError: raise ParsingError( - f"Cannot get {key} because {key_for_error} is not subscriptable." + "Cannot get {} because {} is not subscriptable.".format(key, key_for_error) ) else: result = self.parsed_content[key] return parse_value(result) if parse_types else result - def set(self, key: str, value, set_all: bool = False): - if set_all: + def set(self, key: str, value, all: bool = False): + if all: nested_update(self.parsed_content, key, value, in_place=True) elif "." in key: indexes = split_on_dot(key) @@ -103,8 +104,8 @@ class JsonParser(BaseParser): return True - def delete(self, key, delete_all: bool = False): - if delete_all: + def delete(self, key, all: bool = False): + if all: nested_delete(self.parsed_content, key, in_place=True) elif "." in key: indexes = split_on_dot(key) diff --git a/config_file/parsers/toml_parser.py b/config_file/parsers/toml_parser.py new file mode 100644 index 0000000..e69de29 diff --git a/config_file/parsers/yaml_parser.py b/config_file/parsers/yaml_parser.py new file mode 100644 index 0000000..e69de29 -- cgit v1.2.3 From 8d377a1edf2ac2343fa374cb51a07a72a0316ed2 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 20 Jun 2020 13:47:27 -0500 Subject: Remove taskipy from tox & add default to parser in __determine_parser Taskipy uses f-strings which break on 3.5. --- config_file/config_file.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index 805fc49..1cddd20 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -11,7 +11,9 @@ from config_file.exceptions import ParsingError class ConfigFile: - def __init__(self, file_path: Type[Union[str, PurePath]], parser: Type[BaseParser] = None) -> None: + def __init__( + self, file_path: Type[Union[str, PurePath]], parser: Type[BaseParser] = None + ) -> None: """ Saves the config file path and expands it if needed, reads in the file contents, and determines what parser should be used for @@ -28,7 +30,9 @@ class ConfigFile: self.parser = self.__determine_parser(file_path, parser) @staticmethod - def __create_config_path(file_path: Type[Union[str, PurePath]], original: bool = False) -> str: + def __create_config_path( + file_path: Type[Union[str, PurePath]], original: bool = False + ) -> str: if isinstance(file_path, PurePath): file_path = str(file_path) @@ -41,11 +45,15 @@ class ConfigFile: file_path = ".".join(file_parts) if Path(file_path).is_dir(): - raise ValueError("The specified config file ({}) is a directory.".format(file_path)) + raise ValueError( + "The specified config file ({}) is a directory.".format(file_path) + ) return file_path - def __determine_parser(self, file_path: Type[Union[str, PurePath]], parser: Type[BaseParser]) -> BaseParser: + def __determine_parser( + self, file_path: Type[Union[str, PurePath]], parser: Type[BaseParser] = None + ) -> BaseParser: if isinstance(file_path, PurePath): file_path = str(file_path) @@ -59,7 +67,9 @@ class ConfigFile: return JsonParser(self.contents) else: raise ValueError( - "File path contains an unsupported or unrecognized file type: {}".format(file_path) + "File path contains an unsupported or unrecognized file type: {}".format( + file_path + ) ) def __read_config_file(self): -- cgit v1.2.3 From 95c47a183409fabed81f078a70967f053679737e Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 20 Jun 2020 15:13:53 -0500 Subject: Change path, parser, and contents to be properties Also, restore_original now raises a FileNotFoundError. --- config_file/config_file.py | 94 +++++++++++++++++++++++----------------------- config_file/utils.py | 22 +++++++---- 2 files changed, 61 insertions(+), 55 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index 1cddd20..d4131de 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -1,18 +1,18 @@ import inspect -from pathlib import Path, PurePath +from pathlib import Path from shutil import copyfile from typing import Type, Union +from config_file.exceptions import ParsingError from config_file.parsers.base_parser import BaseParser from config_file.parsers.ini_parser import IniParser from config_file.parsers.json_parser import JsonParser -from config_file.utils import split_on_dot -from config_file.exceptions import ParsingError +from config_file.utils import read_file, split_on_dot class ConfigFile: def __init__( - self, file_path: Type[Union[str, PurePath]], parser: Type[BaseParser] = None + self, file_path: Type[Union[str, Path]], parser: Type[BaseParser] = None ) -> None: """ Saves the config file path and expands it if needed, reads in @@ -25,19 +25,29 @@ class ConfigFile: :raises ValueError: If the specified file path does not have an extension that is supported or it is a directory. """ - self.path = self.__create_config_path(file_path) - self.contents = self.__read_config_file() - self.parser = self.__determine_parser(file_path, parser) + if type(file_path) is not Path: + file_path = Path(file_path) - @staticmethod - def __create_config_path( - file_path: Type[Union[str, PurePath]], original: bool = False - ) -> str: - if isinstance(file_path, PurePath): - file_path = str(file_path) + self.__path = self.__create_config_path(file_path) + self.__contents = read_file(self.__path) + self.__parser = self.__determine_parser(file_path, parser) + + @property + def path(self) -> Path: + return self.__path + + @property + def contents(self) -> str: + return self.__contents - if len(file_path) > 1 and file_path[0] == "~": - return str(Path(file_path).expanduser()) + @property + def parser(self) -> Type[BaseParser]: + return self.__parser + + @staticmethod + def __create_config_path(file_path: Path, original: bool = False) -> str: + if len(file_path.parts) >= 1 and file_path.parts[0] == "~": + return file_path.expanduser() if original: file_parts = split_on_dot(file_path, only_last_dot=True) @@ -52,30 +62,21 @@ class ConfigFile: return file_path def __determine_parser( - self, file_path: Type[Union[str, PurePath]], parser: Type[BaseParser] = None + self, file_path: Path, parser: Type[BaseParser] = None ) -> BaseParser: - if isinstance(file_path, PurePath): - file_path = str(file_path) - if parser is not None and not inspect.isabstract(parser): - return parser(self.contents) + return parser(self.__contents) file_type = split_on_dot(file_path, only_last_dot=True)[-1] if file_type == "ini": - return IniParser(self.contents) + return IniParser(self.__contents) elif file_type == "json": - return JsonParser(self.contents) + return JsonParser(self.__contents) else: raise ValueError( - "File path contains an unsupported or unrecognized file type: {}".format( - file_path - ) + "File path contains an unrecognized file type: {}".format(file_path) ) - def __read_config_file(self): - with open(self.path, "r") as file: - return file.read() - def get(self, key: str, parse_types: bool = False, return_type=None, default=None): """ Retrieve the value of a key. @@ -92,10 +93,10 @@ class ConfigFile: :return: The value of the key. """ if default is None: - key_value = self.parser.get(key, parse_types=parse_types) + key_value = self.__parser.get(key, parse_types=parse_types) else: try: - key_value = self.parser.get(key, parse_types=parse_types) + key_value = self.__parser.get(key, parse_types=parse_types) except ParsingError: return default @@ -103,15 +104,15 @@ class ConfigFile: def set(self, key: str, value): """Sets the value of a key.""" - return self.parser.set(key, value) + return self.__parser.set(key, value) def delete(self, section_key: str): """Deletes a key/value pair or entire sections.""" - return self.parser.delete(section_key) + return self.__parser.delete(section_key) def stringify(self) -> str: """Convert the parsed internal representation of the file back into a string.""" - return self.parser.stringify() + return self.__parser.stringify() def has(self, section_key: str) -> bool: """ @@ -121,7 +122,7 @@ class ConfigFile: Some formats, like JSON, do not have sections and therefore, it would only be checking if the key exists. """ - return self.parser.has(section_key) + return self.__parser.has(section_key) def restore_original(self, original_file_path=None): """ @@ -134,23 +135,22 @@ class ConfigFile: method would look for a config.original.json in the same folder by default. :return: True if the reset succeeded - :raises OSError: If the original configuration file is not present or if an - error occurred when trying to copy the file. + :raises FileNotFoundError: If the original configuration file does not exist + :raises OSError: If self.path is not writable + :raises SameFileError: If self.path and original_file_path are the same file. """ if original_file_path is None: - original_file_path = self.__create_config_path( - str(self.path), original=True - ) + original_file_path = self.__create_config_path(self.__path, original=True) if not Path(original_file_path).exists(): - raise OSError( + raise FileNotFoundError( "The {} file to restore to does not exist.".format(original_file_path) ) - Path(self.path).expanduser().unlink() - copyfile(original_file_path, self.path) - self.contents = self.__read_config_file() - self.parser.parsed_content = self.parser.parse(self.contents) + self.__path.expanduser().unlink() + copyfile(original_file_path, self.__path) + self.__contents = read_file(self.path) + self.__parser.parsed_content = self.__parser.parse(self.__contents) return True @@ -159,7 +159,7 @@ class ConfigFile: Save the configuration changes by writing the file out to the specified path :return: True if the save succeeded. """ - with open(self.path, "w") as config_file: - config_file.write(self.parser.stringify()) + with open(self.__path, "w") as config_file: + config_file.write(self.__parser.stringify()) return True diff --git a/config_file/utils.py b/config_file/utils.py index 61f0ae0..05c3a7a 100644 --- a/config_file/utils.py +++ b/config_file/utils.py @@ -1,4 +1,8 @@ -def split_on_dot(line: str, only_last_dot=False): +from pathlib import Path +from typing import Union + + +def split_on_dot(line: Union[str, Path], only_last_dot=False): """ Split a string on the dot character (.). @@ -7,15 +11,17 @@ def split_on_dot(line: str, only_last_dot=False): :raises ValueError: if the line does not have a dot. """ + if isinstance(line, Path): + line = str(line) + if "." not in line: - # TODO: This is used in the parser for section.keys where a more specific - # error message would be nice, but it is only used more generically to, say, - # split a file path. We may just have to insert some try-catches to catch the - # ValueError and insert a more specific error message or create a custom error - # for this method? raise ValueError( - "section_key must contain the section and key separated by a dot. " - + "e.g. 'section.key'" + "The given string does not have a dot to split on: {}".format(line) ) return line.rsplit(".", 1) if only_last_dot else line.split(".") + + +def read_file(path: Union[str, Path]): + with open(path, "r") as file: + return file.read() -- cgit v1.2.3 From 58808235ea6880c3a0ecfa976a700c81431dc902 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 20 Jun 2020 15:30:11 -0500 Subject: Revert Python 3.5 support --- config_file/config_file.py | 10 +++++----- config_file/parsers/ini_parser.py | 6 +++--- config_file/parsers/json_parser.py | 4 ++-- config_file/utils.py | 14 ++++++-------- 4 files changed, 16 insertions(+), 18 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index d4131de..3af1083 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -55,9 +55,7 @@ class ConfigFile: file_path = ".".join(file_parts) if Path(file_path).is_dir(): - raise ValueError( - "The specified config file ({}) is a directory.".format(file_path) - ) + raise ValueError(f"The specified config file ({file_path}) is a directory.") return file_path @@ -74,7 +72,7 @@ class ConfigFile: return JsonParser(self.__contents) else: raise ValueError( - "File path contains an unrecognized file type: {}".format(file_path) + f"File path contains an unrecognized file type: {file_path}" ) def get(self, key: str, parse_types: bool = False, return_type=None, default=None): @@ -91,6 +89,8 @@ class ConfigFile: :param default: The default value to return if the value of the key is empty. :return: The value of the key. + + :raises ValueError: If the value is not able to be coerced into return_type """ if default is None: key_value = self.__parser.get(key, parse_types=parse_types) @@ -144,7 +144,7 @@ class ConfigFile: if not Path(original_file_path).exists(): raise FileNotFoundError( - "The {} file to restore to does not exist.".format(original_file_path) + f"The {original_file_path} file to restore to does not exist." ) self.__path.expanduser().unlink() diff --git a/config_file/parsers/ini_parser.py b/config_file/parsers/ini_parser.py index 31b0dcf..2db1ae8 100644 --- a/config_file/parsers/ini_parser.py +++ b/config_file/parsers/ini_parser.py @@ -1,10 +1,10 @@ import configparser from io import StringIO +from config_file.exceptions import ParsingError from config_file.parsers.base_parser import BaseParser from config_file.parsers.parse_value import parse_value from config_file.utils import split_on_dot -from config_file.exceptions import ParsingError class IniParser(BaseParser): @@ -96,7 +96,7 @@ class IniParser(BaseParser): if "." not in section_key: if not self.parsed_content.has_section(section_key): raise ValueError( - "Cannot delete {} because it is not in the config file.".format(section_key) + f"Cannot delete {section_key} because it is not in the config file." ) self.parsed_content.remove_section(section_key) @@ -105,7 +105,7 @@ class IniParser(BaseParser): section, key = split_on_dot(section_key, only_last_dot=True) if key not in self.parsed_content[section]: raise ValueError( - "Cannot delete {section}.{key} because {key} is not in {section}.".format(section = section, key = key) + f"Cannot delete {section}.{key} because {key} is not in {section}." ) self.parsed_content.remove_option(section, key) diff --git a/config_file/parsers/json_parser.py b/config_file/parsers/json_parser.py index 4d93ca3..10d5afc 100644 --- a/config_file/parsers/json_parser.py +++ b/config_file/parsers/json_parser.py @@ -7,10 +7,10 @@ from nested_lookup import ( nested_update, ) +from config_file.exceptions import ParsingError from config_file.parsers.base_parser import BaseParser from config_file.parsers.parse_value import parse_value from config_file.utils import split_on_dot -from config_file.exceptions import ParsingError class JsonParser(BaseParser): @@ -78,7 +78,7 @@ class JsonParser(BaseParser): result = result[key] except TypeError: raise ParsingError( - "Cannot get {} because {} is not subscriptable.".format(key, key_for_error) + f"Cannot get {key} because {key_for_error} is not subscriptable." ) else: result = self.parsed_content[key] diff --git a/config_file/utils.py b/config_file/utils.py index 05c3a7a..9ed09b5 100644 --- a/config_file/utils.py +++ b/config_file/utils.py @@ -1,8 +1,8 @@ -from pathlib import Path -from typing import Union +from pathlib import PurePath +from typing import Type, Union -def split_on_dot(line: Union[str, Path], only_last_dot=False): +def split_on_dot(line: Union[str, Type[PurePath]], only_last_dot=False): """ Split a string on the dot character (.). @@ -11,17 +11,15 @@ def split_on_dot(line: Union[str, Path], only_last_dot=False): :raises ValueError: if the line does not have a dot. """ - if isinstance(line, Path): + if isinstance(line, PurePath): line = str(line) if "." not in line: - raise ValueError( - "The given string does not have a dot to split on: {}".format(line) - ) + raise ValueError(f"The given string does not have a dot to split on: {line}") return line.rsplit(".", 1) if only_last_dot else line.split(".") -def read_file(path: Union[str, Path]): +def read_file(path: Union[str, Type[PurePath]]): with open(path, "r") as file: return file.read() -- cgit v1.2.3 From b3fb6aae1201faca1238b3c79e5e95ba57282bf1 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 20 Jun 2020 15:35:05 -0500 Subject: Add more type hints to ConfigFile and lint all files --- config_file/__init__.py | 2 +- config_file/config_file.py | 28 ++++++++++++++++++---------- config_file/exceptions.py | 2 +- config_file/parsers/base_parser.py | 1 - 4 files changed, 20 insertions(+), 13 deletions(-) (limited to 'config_file') diff --git a/config_file/__init__.py b/config_file/__init__.py index f37019a..5fb800a 100644 --- a/config_file/__init__.py +++ b/config_file/__init__.py @@ -1,6 +1,6 @@ from config_file.config_file import ConfigFile +from config_file.exceptions import ConfigFileError, ParsingError from config_file.parsers.base_parser import BaseParser -from config_file.exceptions import ParsingError, ConfigFileError __version__ = "0.10.0" diff --git a/config_file/config_file.py b/config_file/config_file.py index 3af1083..431521a 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -1,7 +1,7 @@ import inspect -from pathlib import Path +from pathlib import Path, PurePath from shutil import copyfile -from typing import Type, Union +from typing import Any, Type, Union from config_file.exceptions import ParsingError from config_file.parsers.base_parser import BaseParser @@ -12,7 +12,9 @@ from config_file.utils import read_file, split_on_dot class ConfigFile: def __init__( - self, file_path: Type[Union[str, Path]], parser: Type[BaseParser] = None + self, + file_path: Type[Union[str, Type[PurePath]]], + parser: Type[BaseParser] = None, ) -> None: """ Saves the config file path and expands it if needed, reads in @@ -33,7 +35,7 @@ class ConfigFile: self.__parser = self.__determine_parser(file_path, parser) @property - def path(self) -> Path: + def path(self) -> Type[PurePath]: return self.__path @property @@ -45,7 +47,7 @@ class ConfigFile: return self.__parser @staticmethod - def __create_config_path(file_path: Path, original: bool = False) -> str: + def __create_config_path(file_path: Type[PurePath], original: bool = False) -> str: if len(file_path.parts) >= 1 and file_path.parts[0] == "~": return file_path.expanduser() @@ -60,7 +62,7 @@ class ConfigFile: return file_path def __determine_parser( - self, file_path: Path, parser: Type[BaseParser] = None + self, file_path: Type[PurePath], parser: Type[BaseParser] = None ) -> BaseParser: if parser is not None and not inspect.isabstract(parser): return parser(self.__contents) @@ -75,7 +77,13 @@ class ConfigFile: f"File path contains an unrecognized file type: {file_path}" ) - def get(self, key: str, parse_types: bool = False, return_type=None, default=None): + def get( + self, + key: str, + parse_types: bool = False, + return_type: Any = None, + default: Any = None, + ) -> Any: """ Retrieve the value of a key. @@ -102,7 +110,7 @@ class ConfigFile: return return_type(key_value) if return_type else key_value - def set(self, key: str, value): + def set(self, key: str, value: Any) -> bool: """Sets the value of a key.""" return self.__parser.set(key, value) @@ -124,7 +132,7 @@ class ConfigFile: """ return self.__parser.has(section_key) - def restore_original(self, original_file_path=None): + def restore_original(self, original_file_path: Union[str, Type[PurePath]] = None): """ Restores the original the config file by deleting it and copying the original back in its place. The internal contents of this config file object are then set @@ -154,7 +162,7 @@ class ConfigFile: return True - def save(self): + def save(self) -> bool: """ Save the configuration changes by writing the file out to the specified path :return: True if the save succeeded. diff --git a/config_file/exceptions.py b/config_file/exceptions.py index cd7e236..e1af755 100644 --- a/config_file/exceptions.py +++ b/config_file/exceptions.py @@ -1,6 +1,6 @@ class ConfigFileError(Exception): """A generic error from the package.""" + class ParsingError(ConfigFileError): """Unable to parse the configuration file.""" - \ No newline at end of file diff --git a/config_file/parsers/base_parser.py b/config_file/parsers/base_parser.py index 5fc47c2..bbdfd91 100644 --- a/config_file/parsers/base_parser.py +++ b/config_file/parsers/base_parser.py @@ -73,4 +73,3 @@ class BaseParser(ABC): if you're going to be using the super() constructor. """ raise NotImplementedError - -- cgit v1.2.3 From ee78049f1e33bcb9894f43d51e44b079edfab91f Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 20 Jun 2020 16:32:23 -0500 Subject: Add types to base parser and update build badge --- config_file/config_file.py | 1 + config_file/parsers/base_parser.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index 431521a..6ded72b 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -83,6 +83,7 @@ class ConfigFile: parse_types: bool = False, return_type: Any = None, default: Any = None, + all: bool = False, ) -> Any: """ Retrieve the value of a key. diff --git a/config_file/parsers/base_parser.py b/config_file/parsers/base_parser.py index bbdfd91..a1d38d4 100644 --- a/config_file/parsers/base_parser.py +++ b/config_file/parsers/base_parser.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from typing import Any class BaseParser(ABC): @@ -25,7 +26,7 @@ class BaseParser(ABC): self.parsed_content = self.parse(self.content) @abstractmethod - def get(self, key, parse_types=False): + def get(self, key: str, parse_types: bool = False, all: bool = False): """ Retrieve the value of a key in its native type. This means the string 'true' will be parsed back as the @@ -37,12 +38,12 @@ class BaseParser(ABC): raise NotImplementedError @abstractmethod - def set(self, key, value) -> bool: + def set(self, key: str, value: Any) -> bool: """Sets the value of a key.""" raise NotImplementedError @abstractmethod - def delete(self, section_key) -> bool: + def delete(self, section_key: str) -> bool: """Deletes a key/value pair or entire sections.""" raise NotImplementedError @@ -66,10 +67,9 @@ class BaseParser(ABC): raise NotImplementedError @abstractmethod - def parse(self, file_contents: str): + def parse(self, file_contents: str) -> Any: """ Parse the file_contents into an internal representation - the given parser can work with. It should not return anything - if you're going to be using the super() constructor. + the given parser can work with. """ raise NotImplementedError -- cgit v1.2.3 From 2c293cc0ac6678253c6af8101ba93866785f64c0 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sun, 21 Jun 2020 11:12:26 -0500 Subject: WIP for refactoring tests --- config_file/config_file.py | 33 ++++----------------------------- config_file/parsers/__init__.py | 6 ++++++ config_file/parsers/json_parser.py | 2 +- config_file/utils.py | 19 +++++++++++++++++-- 4 files changed, 28 insertions(+), 32 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index 6ded72b..e037a82 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -4,10 +4,8 @@ from shutil import copyfile from typing import Any, Type, Union from config_file.exceptions import ParsingError -from config_file.parsers.base_parser import BaseParser -from config_file.parsers.ini_parser import IniParser -from config_file.parsers.json_parser import JsonParser -from config_file.utils import read_file, split_on_dot +from config_file.parsers import BaseParser, IniParser, JsonParser +from config_file.utils import create_config_path, read_file, split_on_dot class ConfigFile: @@ -30,7 +28,7 @@ class ConfigFile: if type(file_path) is not Path: file_path = Path(file_path) - self.__path = self.__create_config_path(file_path) + self.__path = create_config_path(file_path) self.__contents = read_file(self.__path) self.__parser = self.__determine_parser(file_path, parser) @@ -38,29 +36,6 @@ class ConfigFile: def path(self) -> Type[PurePath]: return self.__path - @property - def contents(self) -> str: - return self.__contents - - @property - def parser(self) -> Type[BaseParser]: - return self.__parser - - @staticmethod - def __create_config_path(file_path: Type[PurePath], original: bool = False) -> str: - if len(file_path.parts) >= 1 and file_path.parts[0] == "~": - return file_path.expanduser() - - if original: - file_parts = split_on_dot(file_path, only_last_dot=True) - file_parts.insert(-1, "original") - file_path = ".".join(file_parts) - - if Path(file_path).is_dir(): - raise ValueError(f"The specified config file ({file_path}) is a directory.") - - return file_path - def __determine_parser( self, file_path: Type[PurePath], parser: Type[BaseParser] = None ) -> BaseParser: @@ -149,7 +124,7 @@ class ConfigFile: :raises SameFileError: If self.path and original_file_path are the same file. """ if original_file_path is None: - original_file_path = self.__create_config_path(self.__path, original=True) + original_file_path = create_config_path(self.__path, original=True) if not Path(original_file_path).exists(): raise FileNotFoundError( diff --git a/config_file/parsers/__init__.py b/config_file/parsers/__init__.py index e69de29..415f4f9 100644 --- a/config_file/parsers/__init__.py +++ b/config_file/parsers/__init__.py @@ -0,0 +1,6 @@ +from .base_parser import BaseParser +from .ini_parser import IniParser +from .json_parser import JsonParser +from .parse_value import parse_value + +__all__ = ["BaseParser", "IniParser", "JsonParser", "parse_value"] diff --git a/config_file/parsers/json_parser.py b/config_file/parsers/json_parser.py index 10d5afc..584a2d0 100644 --- a/config_file/parsers/json_parser.py +++ b/config_file/parsers/json_parser.py @@ -124,7 +124,7 @@ class JsonParser(BaseParser): return True def stringify(self) -> str: - return json.dumps(self.parsed_content) + return json.dumps(self.parsed_content, indent=4) def has(self, search_key: str) -> bool: return get_occurrence_of_key(self.parsed_content, key=search_key) > 0 diff --git a/config_file/utils.py b/config_file/utils.py index 9ed09b5..4a83a40 100644 --- a/config_file/utils.py +++ b/config_file/utils.py @@ -1,8 +1,8 @@ -from pathlib import PurePath +from pathlib import Path, PurePath from typing import Type, Union -def split_on_dot(line: Union[str, Type[PurePath]], only_last_dot=False): +def split_on_dot(line: Union[str, Type[PurePath]], only_last_dot=False) -> str: """ Split a string on the dot character (.). @@ -23,3 +23,18 @@ def split_on_dot(line: Union[str, Type[PurePath]], only_last_dot=False): def read_file(path: Union[str, Type[PurePath]]): with open(path, "r") as file: return file.read() + + +def create_config_path(file_path: Type[PurePath], original: bool = False) -> Path: + if len(file_path.parts) >= 1 and file_path.parts[0] == "~": + return file_path.expanduser() + + if original: + file_parts = split_on_dot(file_path, only_last_dot=True) + file_parts.insert(-1, "original") + file_path = Path(".".join(file_parts)) + + if file_path.is_dir(): + raise ValueError(f"The specified config file ({file_path}) is a directory.") + + return file_path -- cgit v1.2.3 From 3a5336c20a983a01bf1495914932f490e97e8d23 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Fri, 10 Jul 2020 20:29:46 -0500 Subject: Add empty yaml and toml parsers --- config_file/parsers/toml_parser.py | 4 ++++ config_file/parsers/yaml_parser.py | 4 ++++ 2 files changed, 8 insertions(+) (limited to 'config_file') diff --git a/config_file/parsers/toml_parser.py b/config_file/parsers/toml_parser.py index e69de29..2b3a48f 100644 --- a/config_file/parsers/toml_parser.py +++ b/config_file/parsers/toml_parser.py @@ -0,0 +1,4 @@ +import toml + +class TomlParser(BaseParser): + pass \ No newline at end of file diff --git a/config_file/parsers/yaml_parser.py b/config_file/parsers/yaml_parser.py index e69de29..e7e88b5 100644 --- a/config_file/parsers/yaml_parser.py +++ b/config_file/parsers/yaml_parser.py @@ -0,0 +1,4 @@ + + +class YamlParser: + pass \ No newline at end of file -- cgit v1.2.3 From fbf798d86b1c2a5d70f39f084ee4e2a376fe6914 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 11 Jul 2020 14:57:24 -0500 Subject: Add dummy toml and yaml parsers --- config_file/parsers/toml_parser.py | 2 ++ config_file/parsers/yaml_parser.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/toml_parser.py b/config_file/parsers/toml_parser.py index 2b3a48f..64ab216 100644 --- a/config_file/parsers/toml_parser.py +++ b/config_file/parsers/toml_parser.py @@ -1,3 +1,5 @@ +from config_file.parsers import BaseParser + import toml class TomlParser(BaseParser): diff --git a/config_file/parsers/yaml_parser.py b/config_file/parsers/yaml_parser.py index e7e88b5..48722b5 100644 --- a/config_file/parsers/yaml_parser.py +++ b/config_file/parsers/yaml_parser.py @@ -1,4 +1,4 @@ +from config_file.parsers import BaseParser - -class YamlParser: - pass \ No newline at end of file +class YamlParser(BaseParser): + pass -- cgit v1.2.3 From 34d81c75ee3e98c0455393a07329296f461cad53 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sun, 12 Jul 2020 09:10:37 -0500 Subject: Add more type hints to ConfigFile --- config_file/config_file.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index e037a82..de8b851 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -25,7 +25,7 @@ class ConfigFile: :raises ValueError: If the specified file path does not have an extension that is supported or it is a directory. """ - if type(file_path) is not Path: + if not isinstance(file_path, Path): file_path = Path(file_path) self.__path = create_config_path(file_path) @@ -38,7 +38,7 @@ class ConfigFile: def __determine_parser( self, file_path: Type[PurePath], parser: Type[BaseParser] = None - ) -> BaseParser: + ) -> Type[BaseParser]: if parser is not None and not inspect.isabstract(parser): return parser(self.__contents) @@ -90,7 +90,7 @@ class ConfigFile: """Sets the value of a key.""" return self.__parser.set(key, value) - def delete(self, section_key: str): + def delete(self, section_key: str) -> bool: """Deletes a key/value pair or entire sections.""" return self.__parser.delete(section_key) @@ -108,7 +108,7 @@ class ConfigFile: """ return self.__parser.has(section_key) - def restore_original(self, original_file_path: Union[str, Type[PurePath]] = None): + def restore_original(self, original_file_path: Union[str, Type[PurePath]] = None) -> bool: """ Restores the original the config file by deleting it and copying the original back in its place. The internal contents of this config file object are then set @@ -133,7 +133,7 @@ class ConfigFile: self.__path.expanduser().unlink() copyfile(original_file_path, self.__path) - self.__contents = read_file(self.path) + self.__contents = read_file(self.__path) self.__parser.parsed_content = self.__parser.parse(self.__contents) return True -- cgit v1.2.3 From 12dd1e7a627b147cfe407c79a09c3477b4a9c623 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sun, 12 Jul 2020 09:11:06 -0500 Subject: Simplify BaseParser import --- config_file/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'config_file') diff --git a/config_file/__init__.py b/config_file/__init__.py index 5fb800a..1b2094f 100644 --- a/config_file/__init__.py +++ b/config_file/__init__.py @@ -1,6 +1,6 @@ from config_file.config_file import ConfigFile from config_file.exceptions import ConfigFileError, ParsingError -from config_file.parsers.base_parser import BaseParser +from config_file.parsers import BaseParser __version__ = "0.10.0" -- cgit v1.2.3 From b467d83e7d67dbda82663634ff60e975f722b115 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sun, 12 Jul 2020 09:11:27 -0500 Subject: Add to __all__ --- config_file/parsers/__init__.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/__init__.py b/config_file/parsers/__init__.py index 415f4f9..8da520e 100644 --- a/config_file/parsers/__init__.py +++ b/config_file/parsers/__init__.py @@ -1,6 +1,23 @@ from .base_parser import BaseParser from .ini_parser import IniParser from .json_parser import JsonParser -from .parse_value import parse_value +from .toml_parser import TomlParser +from .yaml_parser import YamlParser +from .parse_value import ( + parse_value, + can_be_parsed_as_bool, + can_be_parsed_as_float, + can_be_parsed_as_int +) -__all__ = ["BaseParser", "IniParser", "JsonParser", "parse_value"] +__all__ = [ + "BaseParser", + "IniParser", + "JsonParser", + "TomlParser", + "YamlParser", + "parse_value", + "can_be_parsed_as_bool", + "can_be_parsed_as_float", + "can_be_parsed_as_int" +] -- cgit v1.2.3 From c42fd6044f15102d6688cc958a50ab134df046ba Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sun, 12 Jul 2020 09:11:53 -0500 Subject: Add return type hints to BaseParser --- config_file/parsers/base_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/base_parser.py b/config_file/parsers/base_parser.py index a1d38d4..da6e90a 100644 --- a/config_file/parsers/base_parser.py +++ b/config_file/parsers/base_parser.py @@ -12,7 +12,7 @@ class BaseParser(ABC): """ @abstractmethod - def __init__(self, file_contents: str): + def __init__(self, file_contents: str) -> None: """ All parsers will take in the file as a string, parse them internally to their own representation that they @@ -26,7 +26,7 @@ class BaseParser(ABC): self.parsed_content = self.parse(self.content) @abstractmethod - def get(self, key: str, parse_types: bool = False, all: bool = False): + def get(self, key: str, parse_types: bool = False, all: bool = False) -> Any: """ Retrieve the value of a key in its native type. This means the string 'true' will be parsed back as the -- cgit v1.2.3 From 7568a94560b2381f7d7ba77f2c64473cf6842110 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sun, 19 Jul 2020 13:15:24 -0500 Subject: Add more info on installation with poetry and zsh --- config_file/nested-lookup/.gitignore | 11 ++ config_file/nested-lookup/.travis.yml | 9 + config_file/nested-lookup/CONTRIBUTING.rst | 7 + .../nested-lookup/nested_lookup/__init__.py | 8 + .../nested-lookup/nested_lookup/lookup_api.py | 214 +++++++++++++++++++++ .../nested-lookup/nested_lookup/nested_lookup.py | 182 ++++++++++++++++++ config_file/nested_lookup/.gitignore | 11 ++ config_file/nested_lookup/.travis.yml | 9 + config_file/nested_lookup/CONTRIBUTING.rst | 7 + config_file/nested_lookup/__init__.py | 8 + config_file/nested_lookup/lookup_api.py | 214 +++++++++++++++++++++ config_file/nested_lookup/nested_lookup.py | 182 ++++++++++++++++++ .../nested_lookup/nested_lookup/__init__.py | 8 + .../nested_lookup/nested_lookup/lookup_api.py | 214 +++++++++++++++++++++ .../nested_lookup/nested_lookup/nested_lookup.py | 182 ++++++++++++++++++ 15 files changed, 1266 insertions(+) create mode 100644 config_file/nested-lookup/.gitignore create mode 100644 config_file/nested-lookup/.travis.yml create mode 100644 config_file/nested-lookup/CONTRIBUTING.rst create mode 100644 config_file/nested-lookup/nested_lookup/__init__.py create mode 100644 config_file/nested-lookup/nested_lookup/lookup_api.py create mode 100644 config_file/nested-lookup/nested_lookup/nested_lookup.py create mode 100644 config_file/nested_lookup/.gitignore create mode 100644 config_file/nested_lookup/.travis.yml create mode 100644 config_file/nested_lookup/CONTRIBUTING.rst create mode 100644 config_file/nested_lookup/__init__.py create mode 100644 config_file/nested_lookup/lookup_api.py create mode 100644 config_file/nested_lookup/nested_lookup.py create mode 100644 config_file/nested_lookup/nested_lookup/__init__.py create mode 100644 config_file/nested_lookup/nested_lookup/lookup_api.py create mode 100644 config_file/nested_lookup/nested_lookup/nested_lookup.py (limited to 'config_file') diff --git a/config_file/nested-lookup/.gitignore b/config_file/nested-lookup/.gitignore new file mode 100644 index 0000000..c9bd7c4 --- /dev/null +++ b/config_file/nested-lookup/.gitignore @@ -0,0 +1,11 @@ +__pycache__* +.pytest_cache* +*.pyc +.coverage +build/* +dist/* +nested_lookup.egg-info/* +.cache +.vscode +git_commit.sh +venv diff --git a/config_file/nested-lookup/.travis.yml b/config_file/nested-lookup/.travis.yml new file mode 100644 index 0000000..5551955 --- /dev/null +++ b/config_file/nested-lookup/.travis.yml @@ -0,0 +1,9 @@ +language: python +python: + - "2.7" + - "3.6" +# command to install dependencies +install: + - pip install . +# command to run tests +script: pytest \ No newline at end of file diff --git a/config_file/nested-lookup/CONTRIBUTING.rst b/config_file/nested-lookup/CONTRIBUTING.rst new file mode 100644 index 0000000..dde9f01 --- /dev/null +++ b/config_file/nested-lookup/CONTRIBUTING.rst @@ -0,0 +1,7 @@ +The ``nested-lookup`` library is public domain and open source. + +To contribute please make sure all tests pass. + +If you are adding new functionality please write a unit test. + +To avoid discussions regarding code formatting, please use `black `_ with default settings. diff --git a/config_file/nested-lookup/nested_lookup/__init__.py b/config_file/nested-lookup/nested_lookup/__init__.py new file mode 100644 index 0000000..0840df6 --- /dev/null +++ b/config_file/nested-lookup/nested_lookup/__init__.py @@ -0,0 +1,8 @@ +from .nested_lookup import ( + nested_lookup, + get_all_keys, + get_occurrence_of_key, + get_occurrence_of_value, + get_occurrences_and_values +) +from .lookup_api import nested_update, nested_delete, nested_alter diff --git a/config_file/nested-lookup/nested_lookup/lookup_api.py b/config_file/nested-lookup/nested_lookup/lookup_api.py new file mode 100644 index 0000000..a0af63a --- /dev/null +++ b/config_file/nested-lookup/nested_lookup/lookup_api.py @@ -0,0 +1,214 @@ +import copy +import warnings +from six import iteritems +from nested_lookup import nested_lookup + + +def nested_delete(document, key, in_place=False): + if not in_place: + document = copy.deepcopy(document) + return _nested_delete(document=document, key=key) + + +def _nested_delete(document, key): + """ + Method to delete a key->value pair from a nested document + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to delete + Return: + Returns a document that includes everything but the given key + """ + if isinstance(document, list): + for list_items in document: + _nested_delete(document=list_items, key=key) + elif isinstance(document, dict): + if document.get(key): + del document[key] + for dict_key, dict_value in iteritems(document): + _nested_delete(document=dict_value, key=key) + return document + + +def nested_update(document, key, value, in_place=False, treat_as_element=True): + """ + Method to update a key->value pair in a nested document + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to update the value + value: Value to set + in_place (bool): + True: modify the dict in place; + False: create a deep copy of the dict and modify it + Defaults to False + treat_list_element (bool): + True: if a list is provided as "value", the function trys + to match the list elements to the occurences of the key. + If the key occures more often than the provided list has + elements, the first element gets recycled. + False: the provided list is treated as one scalar value and + will be set as value to every key that matches. + Defaults to True (because of backwards portability of the package). + Return: + Returns a document that has updated key, value pair. + """ + + # check if a list or scalar value is provided and create a list + # from the scalar value + # check the length of the list and provide it to _nested_update + if not treat_as_element and not isinstance(value, list): + raise Exception( + "The value must be a list when treat_as_element is False." + ) + elif treat_as_element: + value = [value] + + if not in_place: + document = copy.deepcopy(document) + return _nested_update(document=document, key=key, value=value) + + +def _nested_update(document, key, value): + """ + Method to update a key->value pair in a nested document. + If the number of passed values is less than the number of key matches + when scanning for updates, use last value as default. + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key (str): Key to update the value + value (list): value(s) which should be used for replacement purpouse + Return: + Returns a document that has updated key, value pair. + """ + if isinstance(document, list): + for list_items in document: + _nested_update( + document=list_items, key=key, value=value + ) + elif isinstance(document, dict): + for dict_key, dict_value in iteritems(document): + if dict_key == key: + document[key] = value[0] + if len(value) > 1: + value.pop(0) + _nested_update( + document=dict_value, key=key, value=value + ) + return document + + +def nested_alter( + document, + key, + callback_function=None, + function_parameters=None, + conversion_function=None, + wild_alter=False, + in_place=False, +): + """ + Method to alter all values of the occurences of the key "key". + The provided callback_function is used to alter the scalar values + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to update the value + callback_function :A callback function which alters a scalar value + HINT: You should be aware that not every element might be of + the same type, please check this in your function! + function_parameters (list): + If the callback_function has additional input arguments except + the scalar value, please specify those in this list. + conversion_function: A conversion function like str() which should be + applied to every found value before it is passed to the + "callback_function" + wild_alter: Find matching elements via wild-match by the given keys + and alter those. + HINT: Keep in mind that the wild-match might return unexpected types! + in_place (bool): + True: modify the dict in place; + False: create a deep copy of the dict and modify it + Defaults to False + Return: + Returns a document that has updated key, value pair. + """ + # check if a list or scalar value is provided and create a list from + # the scalar value + # check the length of the list and provide it to _nested_update + if isinstance(key, list): + key_len = len(key) + else: + key = [key] + key_len = len(key) + + if not in_place: + document = copy.deepcopy(document) + return _nested_alter( + document=document, + keys=key, + callback_function=callback_function, + function_parameters=function_parameters, + conversion_function=conversion_function, + wild_alter=wild_alter, + in_place=in_place, + key_len=key_len, + ) + + +def _call_callback( + value_list, callback_function, function_parameters, conversion_function +): + """ + internal helper to call the callback function + """ + return_list = [] + # loop over all values + for value in value_list: + # apply the conversion function + if conversion_function is not None: + value = conversion_function(value) + # if functions arguments are present, expand the list to variables + # via the magic operator * + if function_parameters: + trans_val = callback_function(value, *function_parameters) + else: + trans_val = callback_function(value) + # append the transformed element to the list + return_list.append(trans_val) + return return_list + + +def _nested_alter( + document, + keys, + callback_function, + function_parameters, + conversion_function, + wild_alter, + in_place, + key_len, +): + + # return data if no callback_function is provided + if callback_function is None: + warnings.warn("Please provide a callback_function to nested_alter().") + return document + + # iterate over all given keys in the list + for key in keys: + # try to find the key: + findings = nested_lookup(key, document, with_keys=True, wild=wild_alter) + for k, v in findings.items(): + trans_val = _call_callback( + v, callback_function, function_parameters, conversion_function + ) + # use the transformed value and apply the update to the key + # (dont treat the lists as elements here) + document = nested_update( + document, k, trans_val, in_place=in_place, treat_as_element=False + ) + + return document diff --git a/config_file/nested-lookup/nested_lookup/nested_lookup.py b/config_file/nested-lookup/nested_lookup/nested_lookup.py new file mode 100644 index 0000000..0b11b4d --- /dev/null +++ b/config_file/nested-lookup/nested_lookup/nested_lookup.py @@ -0,0 +1,182 @@ +from six import iteritems + +from collections import defaultdict + + +values_list = [] + + +def nested_lookup(key, document, wild=False, with_keys=False): + """Lookup a key in a nested document, return a list of values""" + if with_keys: + d = defaultdict(list) + for k, v in _nested_lookup(key, document, wild=wild, with_keys=with_keys): + d[k].append(v) + return d + return list(_nested_lookup(key, document, wild=wild, with_keys=with_keys)) + + +def _is_case_insensitive_substring(a, b): + """return True if `a` is a case insensitive substring of `b`, else False""" + return str(a).lower() in str(b).lower() + + +def _nested_lookup(key, document, wild=False, with_keys=False): + """Lookup a key in a nested document, yield a value""" + if isinstance(document, list): + for d in document: + for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): + yield result + + if isinstance(document, dict): + for k, v in iteritems(document): + if key == k or (wild and _is_case_insensitive_substring(key, k)): + if with_keys: + yield k, v + else: + yield v + if isinstance(v, dict): + for result in _nested_lookup(key, v, wild=wild, with_keys=with_keys): + yield result + elif isinstance(v, list): + for d in v: + for result in _nested_lookup( + key, d, wild=wild, with_keys=with_keys + ): + yield result + + +def get_all_keys(dictionary): + """ + Method to get all keys from a nested dictionary as a List + Args: + dictionary: Nested dictionary + Returns: + List of keys in the dictionary + """ + result_list = [] + + def recrusion(document): + if isinstance(document, list): + for list_items in document: + recrusion(document=list_items) + elif isinstance(document, dict): + for key, value in iteritems(document): + result_list.append(key) + recrusion(document=value) + return + + recrusion(document=dictionary) + return result_list + + +def get_occurrence_of_key(dictionary, key): + """ + Method to get occurrence of a key in a nested dictionary + + Args: + dictionary: Nested dictionary + key: Key to search for the occurrences + Return: + Number of occurrence (Integer) + """ + return _get_occurrence(dictionary=dictionary, item="key", keyword=key) + + +def get_occurrences_and_values(items, value): + """ + Method to get occurrence of a value in a nested list of dictionary + + Args: + items: list of dictionary: Nested dictionary + value: Value to search for the occurrences + + Return: + Dict where the key is the value arg and his value is a new + dict with occurrences and values + """ + occurrences = {} + occurrence = 0 + value_list = [] + + for item in items: + occurrence_result, values = _get_occurrence_with_values(dictionary=item, item="value", keyword=value) + occurrence = occurrence + occurrence_result + if occurrence_result: + value_list.extend(values) + + occurrences[value] = { + 'occurrences': occurrence, + 'values': value_list + } + + return occurrences + + +def _get_occurrence_with_values(dictionary, item, keyword): + occurrence = [0] + + result_recursion = _recursion(dictionary, item, keyword, occurrence, True) + + global values_list + values_list = [] + + return occurrence[0], result_recursion + + +def get_occurrence_of_value(dictionary, value): + """ + Method to get occurrence of a value in a nested dictionary + + Args: + dictionary: Nested dictionary + value: Value to search for the occurrences + Return: + Number of occurrence (Integer) + """ + return _get_occurrence(dictionary=dictionary, item="value", keyword=value) + + +def _recursion(dictionary, item, keyword, occurrence, with_values=False): + + global values_list + + if item == "key": + if dictionary.get(keyword) is not None: + occurrence[0] += 1 + elif keyword in list(dictionary.values()): + occurrence[0] += list(dictionary.values()).count(keyword) + if with_values: + values_list.append(dictionary) + for key, value in iteritems(dictionary): + if isinstance(value, dict): + _recursion(value, item, keyword, occurrence, with_values) + elif isinstance(value, list): + for list_items in value: + if hasattr(list_items, "items"): + _recursion(list_items, item, keyword, occurrence, with_values) + elif list_items == keyword: + occurrence[0] += 1 if item == "value" else 0 + + if values_list: + return values_list + + +def _get_occurrence(dictionary, item, keyword): + """ + Method to get occurrence of a key or value in a nested dictionary + + Args: + dictionary: Nested dictionary + item: Mostly (key or value) + keyword: key word to find occurrence + Return: + Number of occurrence of the given keyword in the dict + """ + occurrence = [0] + _recursion(dictionary, item, keyword, occurrence) + + global values_list + values_list = [] + + return occurrence[0] diff --git a/config_file/nested_lookup/.gitignore b/config_file/nested_lookup/.gitignore new file mode 100644 index 0000000..c9bd7c4 --- /dev/null +++ b/config_file/nested_lookup/.gitignore @@ -0,0 +1,11 @@ +__pycache__* +.pytest_cache* +*.pyc +.coverage +build/* +dist/* +nested_lookup.egg-info/* +.cache +.vscode +git_commit.sh +venv diff --git a/config_file/nested_lookup/.travis.yml b/config_file/nested_lookup/.travis.yml new file mode 100644 index 0000000..5551955 --- /dev/null +++ b/config_file/nested_lookup/.travis.yml @@ -0,0 +1,9 @@ +language: python +python: + - "2.7" + - "3.6" +# command to install dependencies +install: + - pip install . +# command to run tests +script: pytest \ No newline at end of file diff --git a/config_file/nested_lookup/CONTRIBUTING.rst b/config_file/nested_lookup/CONTRIBUTING.rst new file mode 100644 index 0000000..dde9f01 --- /dev/null +++ b/config_file/nested_lookup/CONTRIBUTING.rst @@ -0,0 +1,7 @@ +The ``nested-lookup`` library is public domain and open source. + +To contribute please make sure all tests pass. + +If you are adding new functionality please write a unit test. + +To avoid discussions regarding code formatting, please use `black `_ with default settings. diff --git a/config_file/nested_lookup/__init__.py b/config_file/nested_lookup/__init__.py new file mode 100644 index 0000000..0840df6 --- /dev/null +++ b/config_file/nested_lookup/__init__.py @@ -0,0 +1,8 @@ +from .nested_lookup import ( + nested_lookup, + get_all_keys, + get_occurrence_of_key, + get_occurrence_of_value, + get_occurrences_and_values +) +from .lookup_api import nested_update, nested_delete, nested_alter diff --git a/config_file/nested_lookup/lookup_api.py b/config_file/nested_lookup/lookup_api.py new file mode 100644 index 0000000..a0af63a --- /dev/null +++ b/config_file/nested_lookup/lookup_api.py @@ -0,0 +1,214 @@ +import copy +import warnings +from six import iteritems +from nested_lookup import nested_lookup + + +def nested_delete(document, key, in_place=False): + if not in_place: + document = copy.deepcopy(document) + return _nested_delete(document=document, key=key) + + +def _nested_delete(document, key): + """ + Method to delete a key->value pair from a nested document + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to delete + Return: + Returns a document that includes everything but the given key + """ + if isinstance(document, list): + for list_items in document: + _nested_delete(document=list_items, key=key) + elif isinstance(document, dict): + if document.get(key): + del document[key] + for dict_key, dict_value in iteritems(document): + _nested_delete(document=dict_value, key=key) + return document + + +def nested_update(document, key, value, in_place=False, treat_as_element=True): + """ + Method to update a key->value pair in a nested document + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to update the value + value: Value to set + in_place (bool): + True: modify the dict in place; + False: create a deep copy of the dict and modify it + Defaults to False + treat_list_element (bool): + True: if a list is provided as "value", the function trys + to match the list elements to the occurences of the key. + If the key occures more often than the provided list has + elements, the first element gets recycled. + False: the provided list is treated as one scalar value and + will be set as value to every key that matches. + Defaults to True (because of backwards portability of the package). + Return: + Returns a document that has updated key, value pair. + """ + + # check if a list or scalar value is provided and create a list + # from the scalar value + # check the length of the list and provide it to _nested_update + if not treat_as_element and not isinstance(value, list): + raise Exception( + "The value must be a list when treat_as_element is False." + ) + elif treat_as_element: + value = [value] + + if not in_place: + document = copy.deepcopy(document) + return _nested_update(document=document, key=key, value=value) + + +def _nested_update(document, key, value): + """ + Method to update a key->value pair in a nested document. + If the number of passed values is less than the number of key matches + when scanning for updates, use last value as default. + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key (str): Key to update the value + value (list): value(s) which should be used for replacement purpouse + Return: + Returns a document that has updated key, value pair. + """ + if isinstance(document, list): + for list_items in document: + _nested_update( + document=list_items, key=key, value=value + ) + elif isinstance(document, dict): + for dict_key, dict_value in iteritems(document): + if dict_key == key: + document[key] = value[0] + if len(value) > 1: + value.pop(0) + _nested_update( + document=dict_value, key=key, value=value + ) + return document + + +def nested_alter( + document, + key, + callback_function=None, + function_parameters=None, + conversion_function=None, + wild_alter=False, + in_place=False, +): + """ + Method to alter all values of the occurences of the key "key". + The provided callback_function is used to alter the scalar values + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to update the value + callback_function :A callback function which alters a scalar value + HINT: You should be aware that not every element might be of + the same type, please check this in your function! + function_parameters (list): + If the callback_function has additional input arguments except + the scalar value, please specify those in this list. + conversion_function: A conversion function like str() which should be + applied to every found value before it is passed to the + "callback_function" + wild_alter: Find matching elements via wild-match by the given keys + and alter those. + HINT: Keep in mind that the wild-match might return unexpected types! + in_place (bool): + True: modify the dict in place; + False: create a deep copy of the dict and modify it + Defaults to False + Return: + Returns a document that has updated key, value pair. + """ + # check if a list or scalar value is provided and create a list from + # the scalar value + # check the length of the list and provide it to _nested_update + if isinstance(key, list): + key_len = len(key) + else: + key = [key] + key_len = len(key) + + if not in_place: + document = copy.deepcopy(document) + return _nested_alter( + document=document, + keys=key, + callback_function=callback_function, + function_parameters=function_parameters, + conversion_function=conversion_function, + wild_alter=wild_alter, + in_place=in_place, + key_len=key_len, + ) + + +def _call_callback( + value_list, callback_function, function_parameters, conversion_function +): + """ + internal helper to call the callback function + """ + return_list = [] + # loop over all values + for value in value_list: + # apply the conversion function + if conversion_function is not None: + value = conversion_function(value) + # if functions arguments are present, expand the list to variables + # via the magic operator * + if function_parameters: + trans_val = callback_function(value, *function_parameters) + else: + trans_val = callback_function(value) + # append the transformed element to the list + return_list.append(trans_val) + return return_list + + +def _nested_alter( + document, + keys, + callback_function, + function_parameters, + conversion_function, + wild_alter, + in_place, + key_len, +): + + # return data if no callback_function is provided + if callback_function is None: + warnings.warn("Please provide a callback_function to nested_alter().") + return document + + # iterate over all given keys in the list + for key in keys: + # try to find the key: + findings = nested_lookup(key, document, with_keys=True, wild=wild_alter) + for k, v in findings.items(): + trans_val = _call_callback( + v, callback_function, function_parameters, conversion_function + ) + # use the transformed value and apply the update to the key + # (dont treat the lists as elements here) + document = nested_update( + document, k, trans_val, in_place=in_place, treat_as_element=False + ) + + return document diff --git a/config_file/nested_lookup/nested_lookup.py b/config_file/nested_lookup/nested_lookup.py new file mode 100644 index 0000000..0b11b4d --- /dev/null +++ b/config_file/nested_lookup/nested_lookup.py @@ -0,0 +1,182 @@ +from six import iteritems + +from collections import defaultdict + + +values_list = [] + + +def nested_lookup(key, document, wild=False, with_keys=False): + """Lookup a key in a nested document, return a list of values""" + if with_keys: + d = defaultdict(list) + for k, v in _nested_lookup(key, document, wild=wild, with_keys=with_keys): + d[k].append(v) + return d + return list(_nested_lookup(key, document, wild=wild, with_keys=with_keys)) + + +def _is_case_insensitive_substring(a, b): + """return True if `a` is a case insensitive substring of `b`, else False""" + return str(a).lower() in str(b).lower() + + +def _nested_lookup(key, document, wild=False, with_keys=False): + """Lookup a key in a nested document, yield a value""" + if isinstance(document, list): + for d in document: + for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): + yield result + + if isinstance(document, dict): + for k, v in iteritems(document): + if key == k or (wild and _is_case_insensitive_substring(key, k)): + if with_keys: + yield k, v + else: + yield v + if isinstance(v, dict): + for result in _nested_lookup(key, v, wild=wild, with_keys=with_keys): + yield result + elif isinstance(v, list): + for d in v: + for result in _nested_lookup( + key, d, wild=wild, with_keys=with_keys + ): + yield result + + +def get_all_keys(dictionary): + """ + Method to get all keys from a nested dictionary as a List + Args: + dictionary: Nested dictionary + Returns: + List of keys in the dictionary + """ + result_list = [] + + def recrusion(document): + if isinstance(document, list): + for list_items in document: + recrusion(document=list_items) + elif isinstance(document, dict): + for key, value in iteritems(document): + result_list.append(key) + recrusion(document=value) + return + + recrusion(document=dictionary) + return result_list + + +def get_occurrence_of_key(dictionary, key): + """ + Method to get occurrence of a key in a nested dictionary + + Args: + dictionary: Nested dictionary + key: Key to search for the occurrences + Return: + Number of occurrence (Integer) + """ + return _get_occurrence(dictionary=dictionary, item="key", keyword=key) + + +def get_occurrences_and_values(items, value): + """ + Method to get occurrence of a value in a nested list of dictionary + + Args: + items: list of dictionary: Nested dictionary + value: Value to search for the occurrences + + Return: + Dict where the key is the value arg and his value is a new + dict with occurrences and values + """ + occurrences = {} + occurrence = 0 + value_list = [] + + for item in items: + occurrence_result, values = _get_occurrence_with_values(dictionary=item, item="value", keyword=value) + occurrence = occurrence + occurrence_result + if occurrence_result: + value_list.extend(values) + + occurrences[value] = { + 'occurrences': occurrence, + 'values': value_list + } + + return occurrences + + +def _get_occurrence_with_values(dictionary, item, keyword): + occurrence = [0] + + result_recursion = _recursion(dictionary, item, keyword, occurrence, True) + + global values_list + values_list = [] + + return occurrence[0], result_recursion + + +def get_occurrence_of_value(dictionary, value): + """ + Method to get occurrence of a value in a nested dictionary + + Args: + dictionary: Nested dictionary + value: Value to search for the occurrences + Return: + Number of occurrence (Integer) + """ + return _get_occurrence(dictionary=dictionary, item="value", keyword=value) + + +def _recursion(dictionary, item, keyword, occurrence, with_values=False): + + global values_list + + if item == "key": + if dictionary.get(keyword) is not None: + occurrence[0] += 1 + elif keyword in list(dictionary.values()): + occurrence[0] += list(dictionary.values()).count(keyword) + if with_values: + values_list.append(dictionary) + for key, value in iteritems(dictionary): + if isinstance(value, dict): + _recursion(value, item, keyword, occurrence, with_values) + elif isinstance(value, list): + for list_items in value: + if hasattr(list_items, "items"): + _recursion(list_items, item, keyword, occurrence, with_values) + elif list_items == keyword: + occurrence[0] += 1 if item == "value" else 0 + + if values_list: + return values_list + + +def _get_occurrence(dictionary, item, keyword): + """ + Method to get occurrence of a key or value in a nested dictionary + + Args: + dictionary: Nested dictionary + item: Mostly (key or value) + keyword: key word to find occurrence + Return: + Number of occurrence of the given keyword in the dict + """ + occurrence = [0] + _recursion(dictionary, item, keyword, occurrence) + + global values_list + values_list = [] + + return occurrence[0] diff --git a/config_file/nested_lookup/nested_lookup/__init__.py b/config_file/nested_lookup/nested_lookup/__init__.py new file mode 100644 index 0000000..0840df6 --- /dev/null +++ b/config_file/nested_lookup/nested_lookup/__init__.py @@ -0,0 +1,8 @@ +from .nested_lookup import ( + nested_lookup, + get_all_keys, + get_occurrence_of_key, + get_occurrence_of_value, + get_occurrences_and_values +) +from .lookup_api import nested_update, nested_delete, nested_alter diff --git a/config_file/nested_lookup/nested_lookup/lookup_api.py b/config_file/nested_lookup/nested_lookup/lookup_api.py new file mode 100644 index 0000000..a0af63a --- /dev/null +++ b/config_file/nested_lookup/nested_lookup/lookup_api.py @@ -0,0 +1,214 @@ +import copy +import warnings +from six import iteritems +from nested_lookup import nested_lookup + + +def nested_delete(document, key, in_place=False): + if not in_place: + document = copy.deepcopy(document) + return _nested_delete(document=document, key=key) + + +def _nested_delete(document, key): + """ + Method to delete a key->value pair from a nested document + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to delete + Return: + Returns a document that includes everything but the given key + """ + if isinstance(document, list): + for list_items in document: + _nested_delete(document=list_items, key=key) + elif isinstance(document, dict): + if document.get(key): + del document[key] + for dict_key, dict_value in iteritems(document): + _nested_delete(document=dict_value, key=key) + return document + + +def nested_update(document, key, value, in_place=False, treat_as_element=True): + """ + Method to update a key->value pair in a nested document + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to update the value + value: Value to set + in_place (bool): + True: modify the dict in place; + False: create a deep copy of the dict and modify it + Defaults to False + treat_list_element (bool): + True: if a list is provided as "value", the function trys + to match the list elements to the occurences of the key. + If the key occures more often than the provided list has + elements, the first element gets recycled. + False: the provided list is treated as one scalar value and + will be set as value to every key that matches. + Defaults to True (because of backwards portability of the package). + Return: + Returns a document that has updated key, value pair. + """ + + # check if a list or scalar value is provided and create a list + # from the scalar value + # check the length of the list and provide it to _nested_update + if not treat_as_element and not isinstance(value, list): + raise Exception( + "The value must be a list when treat_as_element is False." + ) + elif treat_as_element: + value = [value] + + if not in_place: + document = copy.deepcopy(document) + return _nested_update(document=document, key=key, value=value) + + +def _nested_update(document, key, value): + """ + Method to update a key->value pair in a nested document. + If the number of passed values is less than the number of key matches + when scanning for updates, use last value as default. + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key (str): Key to update the value + value (list): value(s) which should be used for replacement purpouse + Return: + Returns a document that has updated key, value pair. + """ + if isinstance(document, list): + for list_items in document: + _nested_update( + document=list_items, key=key, value=value + ) + elif isinstance(document, dict): + for dict_key, dict_value in iteritems(document): + if dict_key == key: + document[key] = value[0] + if len(value) > 1: + value.pop(0) + _nested_update( + document=dict_value, key=key, value=value + ) + return document + + +def nested_alter( + document, + key, + callback_function=None, + function_parameters=None, + conversion_function=None, + wild_alter=False, + in_place=False, +): + """ + Method to alter all values of the occurences of the key "key". + The provided callback_function is used to alter the scalar values + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to update the value + callback_function :A callback function which alters a scalar value + HINT: You should be aware that not every element might be of + the same type, please check this in your function! + function_parameters (list): + If the callback_function has additional input arguments except + the scalar value, please specify those in this list. + conversion_function: A conversion function like str() which should be + applied to every found value before it is passed to the + "callback_function" + wild_alter: Find matching elements via wild-match by the given keys + and alter those. + HINT: Keep in mind that the wild-match might return unexpected types! + in_place (bool): + True: modify the dict in place; + False: create a deep copy of the dict and modify it + Defaults to False + Return: + Returns a document that has updated key, value pair. + """ + # check if a list or scalar value is provided and create a list from + # the scalar value + # check the length of the list and provide it to _nested_update + if isinstance(key, list): + key_len = len(key) + else: + key = [key] + key_len = len(key) + + if not in_place: + document = copy.deepcopy(document) + return _nested_alter( + document=document, + keys=key, + callback_function=callback_function, + function_parameters=function_parameters, + conversion_function=conversion_function, + wild_alter=wild_alter, + in_place=in_place, + key_len=key_len, + ) + + +def _call_callback( + value_list, callback_function, function_parameters, conversion_function +): + """ + internal helper to call the callback function + """ + return_list = [] + # loop over all values + for value in value_list: + # apply the conversion function + if conversion_function is not None: + value = conversion_function(value) + # if functions arguments are present, expand the list to variables + # via the magic operator * + if function_parameters: + trans_val = callback_function(value, *function_parameters) + else: + trans_val = callback_function(value) + # append the transformed element to the list + return_list.append(trans_val) + return return_list + + +def _nested_alter( + document, + keys, + callback_function, + function_parameters, + conversion_function, + wild_alter, + in_place, + key_len, +): + + # return data if no callback_function is provided + if callback_function is None: + warnings.warn("Please provide a callback_function to nested_alter().") + return document + + # iterate over all given keys in the list + for key in keys: + # try to find the key: + findings = nested_lookup(key, document, with_keys=True, wild=wild_alter) + for k, v in findings.items(): + trans_val = _call_callback( + v, callback_function, function_parameters, conversion_function + ) + # use the transformed value and apply the update to the key + # (dont treat the lists as elements here) + document = nested_update( + document, k, trans_val, in_place=in_place, treat_as_element=False + ) + + return document diff --git a/config_file/nested_lookup/nested_lookup/nested_lookup.py b/config_file/nested_lookup/nested_lookup/nested_lookup.py new file mode 100644 index 0000000..0b11b4d --- /dev/null +++ b/config_file/nested_lookup/nested_lookup/nested_lookup.py @@ -0,0 +1,182 @@ +from six import iteritems + +from collections import defaultdict + + +values_list = [] + + +def nested_lookup(key, document, wild=False, with_keys=False): + """Lookup a key in a nested document, return a list of values""" + if with_keys: + d = defaultdict(list) + for k, v in _nested_lookup(key, document, wild=wild, with_keys=with_keys): + d[k].append(v) + return d + return list(_nested_lookup(key, document, wild=wild, with_keys=with_keys)) + + +def _is_case_insensitive_substring(a, b): + """return True if `a` is a case insensitive substring of `b`, else False""" + return str(a).lower() in str(b).lower() + + +def _nested_lookup(key, document, wild=False, with_keys=False): + """Lookup a key in a nested document, yield a value""" + if isinstance(document, list): + for d in document: + for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): + yield result + + if isinstance(document, dict): + for k, v in iteritems(document): + if key == k or (wild and _is_case_insensitive_substring(key, k)): + if with_keys: + yield k, v + else: + yield v + if isinstance(v, dict): + for result in _nested_lookup(key, v, wild=wild, with_keys=with_keys): + yield result + elif isinstance(v, list): + for d in v: + for result in _nested_lookup( + key, d, wild=wild, with_keys=with_keys + ): + yield result + + +def get_all_keys(dictionary): + """ + Method to get all keys from a nested dictionary as a List + Args: + dictionary: Nested dictionary + Returns: + List of keys in the dictionary + """ + result_list = [] + + def recrusion(document): + if isinstance(document, list): + for list_items in document: + recrusion(document=list_items) + elif isinstance(document, dict): + for key, value in iteritems(document): + result_list.append(key) + recrusion(document=value) + return + + recrusion(document=dictionary) + return result_list + + +def get_occurrence_of_key(dictionary, key): + """ + Method to get occurrence of a key in a nested dictionary + + Args: + dictionary: Nested dictionary + key: Key to search for the occurrences + Return: + Number of occurrence (Integer) + """ + return _get_occurrence(dictionary=dictionary, item="key", keyword=key) + + +def get_occurrences_and_values(items, value): + """ + Method to get occurrence of a value in a nested list of dictionary + + Args: + items: list of dictionary: Nested dictionary + value: Value to search for the occurrences + + Return: + Dict where the key is the value arg and his value is a new + dict with occurrences and values + """ + occurrences = {} + occurrence = 0 + value_list = [] + + for item in items: + occurrence_result, values = _get_occurrence_with_values(dictionary=item, item="value", keyword=value) + occurrence = occurrence + occurrence_result + if occurrence_result: + value_list.extend(values) + + occurrences[value] = { + 'occurrences': occurrence, + 'values': value_list + } + + return occurrences + + +def _get_occurrence_with_values(dictionary, item, keyword): + occurrence = [0] + + result_recursion = _recursion(dictionary, item, keyword, occurrence, True) + + global values_list + values_list = [] + + return occurrence[0], result_recursion + + +def get_occurrence_of_value(dictionary, value): + """ + Method to get occurrence of a value in a nested dictionary + + Args: + dictionary: Nested dictionary + value: Value to search for the occurrences + Return: + Number of occurrence (Integer) + """ + return _get_occurrence(dictionary=dictionary, item="value", keyword=value) + + +def _recursion(dictionary, item, keyword, occurrence, with_values=False): + + global values_list + + if item == "key": + if dictionary.get(keyword) is not None: + occurrence[0] += 1 + elif keyword in list(dictionary.values()): + occurrence[0] += list(dictionary.values()).count(keyword) + if with_values: + values_list.append(dictionary) + for key, value in iteritems(dictionary): + if isinstance(value, dict): + _recursion(value, item, keyword, occurrence, with_values) + elif isinstance(value, list): + for list_items in value: + if hasattr(list_items, "items"): + _recursion(list_items, item, keyword, occurrence, with_values) + elif list_items == keyword: + occurrence[0] += 1 if item == "value" else 0 + + if values_list: + return values_list + + +def _get_occurrence(dictionary, item, keyword): + """ + Method to get occurrence of a key or value in a nested dictionary + + Args: + dictionary: Nested dictionary + item: Mostly (key or value) + keyword: key word to find occurrence + Return: + Number of occurrence of the given keyword in the dict + """ + occurrence = [0] + _recursion(dictionary, item, keyword, occurrence) + + global values_list + values_list = [] + + return occurrence[0] -- cgit v1.2.3 From bc1280444ea3d5d4e5b5afd4e39bfb9365689393 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sun, 19 Jul 2020 13:17:09 -0500 Subject: Add nested-lookup to project russellballestrini is a python project in the public domain. Rather than installing it as a dependency, this adds it as part of the config-file package now, which allows more control. --- config_file/nested-lookup/.gitignore | 11 -- config_file/nested-lookup/.travis.yml | 9 - config_file/nested-lookup/CONTRIBUTING.rst | 7 - .../nested-lookup/nested_lookup/__init__.py | 8 - .../nested-lookup/nested_lookup/lookup_api.py | 214 --------------------- .../nested-lookup/nested_lookup/nested_lookup.py | 182 ------------------ config_file/nested_lookup/.gitignore | 11 -- config_file/nested_lookup/.travis.yml | 9 - config_file/nested_lookup/CONTRIBUTING.rst | 7 - config_file/nested_lookup/__init__.py | 6 +- config_file/nested_lookup/lookup_api.py | 16 +- config_file/nested_lookup/nested_lookup.py | 8 +- .../nested_lookup/nested_lookup/__init__.py | 8 - .../nested_lookup/nested_lookup/lookup_api.py | 214 --------------------- .../nested_lookup/nested_lookup/nested_lookup.py | 182 ------------------ config_file/parsers/json_parser.py | 12 +- config_file/parsers/toml_parser.py | 105 +++++++++- 17 files changed, 126 insertions(+), 883 deletions(-) delete mode 100644 config_file/nested-lookup/.gitignore delete mode 100644 config_file/nested-lookup/.travis.yml delete mode 100644 config_file/nested-lookup/CONTRIBUTING.rst delete mode 100644 config_file/nested-lookup/nested_lookup/__init__.py delete mode 100644 config_file/nested-lookup/nested_lookup/lookup_api.py delete mode 100644 config_file/nested-lookup/nested_lookup/nested_lookup.py delete mode 100644 config_file/nested_lookup/.gitignore delete mode 100644 config_file/nested_lookup/.travis.yml delete mode 100644 config_file/nested_lookup/CONTRIBUTING.rst delete mode 100644 config_file/nested_lookup/nested_lookup/__init__.py delete mode 100644 config_file/nested_lookup/nested_lookup/lookup_api.py delete mode 100644 config_file/nested_lookup/nested_lookup/nested_lookup.py (limited to 'config_file') diff --git a/config_file/nested-lookup/.gitignore b/config_file/nested-lookup/.gitignore deleted file mode 100644 index c9bd7c4..0000000 --- a/config_file/nested-lookup/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -__pycache__* -.pytest_cache* -*.pyc -.coverage -build/* -dist/* -nested_lookup.egg-info/* -.cache -.vscode -git_commit.sh -venv diff --git a/config_file/nested-lookup/.travis.yml b/config_file/nested-lookup/.travis.yml deleted file mode 100644 index 5551955..0000000 --- a/config_file/nested-lookup/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -language: python -python: - - "2.7" - - "3.6" -# command to install dependencies -install: - - pip install . -# command to run tests -script: pytest \ No newline at end of file diff --git a/config_file/nested-lookup/CONTRIBUTING.rst b/config_file/nested-lookup/CONTRIBUTING.rst deleted file mode 100644 index dde9f01..0000000 --- a/config_file/nested-lookup/CONTRIBUTING.rst +++ /dev/null @@ -1,7 +0,0 @@ -The ``nested-lookup`` library is public domain and open source. - -To contribute please make sure all tests pass. - -If you are adding new functionality please write a unit test. - -To avoid discussions regarding code formatting, please use `black `_ with default settings. diff --git a/config_file/nested-lookup/nested_lookup/__init__.py b/config_file/nested-lookup/nested_lookup/__init__.py deleted file mode 100644 index 0840df6..0000000 --- a/config_file/nested-lookup/nested_lookup/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .nested_lookup import ( - nested_lookup, - get_all_keys, - get_occurrence_of_key, - get_occurrence_of_value, - get_occurrences_and_values -) -from .lookup_api import nested_update, nested_delete, nested_alter diff --git a/config_file/nested-lookup/nested_lookup/lookup_api.py b/config_file/nested-lookup/nested_lookup/lookup_api.py deleted file mode 100644 index a0af63a..0000000 --- a/config_file/nested-lookup/nested_lookup/lookup_api.py +++ /dev/null @@ -1,214 +0,0 @@ -import copy -import warnings -from six import iteritems -from nested_lookup import nested_lookup - - -def nested_delete(document, key, in_place=False): - if not in_place: - document = copy.deepcopy(document) - return _nested_delete(document=document, key=key) - - -def _nested_delete(document, key): - """ - Method to delete a key->value pair from a nested document - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key: Key to delete - Return: - Returns a document that includes everything but the given key - """ - if isinstance(document, list): - for list_items in document: - _nested_delete(document=list_items, key=key) - elif isinstance(document, dict): - if document.get(key): - del document[key] - for dict_key, dict_value in iteritems(document): - _nested_delete(document=dict_value, key=key) - return document - - -def nested_update(document, key, value, in_place=False, treat_as_element=True): - """ - Method to update a key->value pair in a nested document - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key: Key to update the value - value: Value to set - in_place (bool): - True: modify the dict in place; - False: create a deep copy of the dict and modify it - Defaults to False - treat_list_element (bool): - True: if a list is provided as "value", the function trys - to match the list elements to the occurences of the key. - If the key occures more often than the provided list has - elements, the first element gets recycled. - False: the provided list is treated as one scalar value and - will be set as value to every key that matches. - Defaults to True (because of backwards portability of the package). - Return: - Returns a document that has updated key, value pair. - """ - - # check if a list or scalar value is provided and create a list - # from the scalar value - # check the length of the list and provide it to _nested_update - if not treat_as_element and not isinstance(value, list): - raise Exception( - "The value must be a list when treat_as_element is False." - ) - elif treat_as_element: - value = [value] - - if not in_place: - document = copy.deepcopy(document) - return _nested_update(document=document, key=key, value=value) - - -def _nested_update(document, key, value): - """ - Method to update a key->value pair in a nested document. - If the number of passed values is less than the number of key matches - when scanning for updates, use last value as default. - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key (str): Key to update the value - value (list): value(s) which should be used for replacement purpouse - Return: - Returns a document that has updated key, value pair. - """ - if isinstance(document, list): - for list_items in document: - _nested_update( - document=list_items, key=key, value=value - ) - elif isinstance(document, dict): - for dict_key, dict_value in iteritems(document): - if dict_key == key: - document[key] = value[0] - if len(value) > 1: - value.pop(0) - _nested_update( - document=dict_value, key=key, value=value - ) - return document - - -def nested_alter( - document, - key, - callback_function=None, - function_parameters=None, - conversion_function=None, - wild_alter=False, - in_place=False, -): - """ - Method to alter all values of the occurences of the key "key". - The provided callback_function is used to alter the scalar values - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key: Key to update the value - callback_function :A callback function which alters a scalar value - HINT: You should be aware that not every element might be of - the same type, please check this in your function! - function_parameters (list): - If the callback_function has additional input arguments except - the scalar value, please specify those in this list. - conversion_function: A conversion function like str() which should be - applied to every found value before it is passed to the - "callback_function" - wild_alter: Find matching elements via wild-match by the given keys - and alter those. - HINT: Keep in mind that the wild-match might return unexpected types! - in_place (bool): - True: modify the dict in place; - False: create a deep copy of the dict and modify it - Defaults to False - Return: - Returns a document that has updated key, value pair. - """ - # check if a list or scalar value is provided and create a list from - # the scalar value - # check the length of the list and provide it to _nested_update - if isinstance(key, list): - key_len = len(key) - else: - key = [key] - key_len = len(key) - - if not in_place: - document = copy.deepcopy(document) - return _nested_alter( - document=document, - keys=key, - callback_function=callback_function, - function_parameters=function_parameters, - conversion_function=conversion_function, - wild_alter=wild_alter, - in_place=in_place, - key_len=key_len, - ) - - -def _call_callback( - value_list, callback_function, function_parameters, conversion_function -): - """ - internal helper to call the callback function - """ - return_list = [] - # loop over all values - for value in value_list: - # apply the conversion function - if conversion_function is not None: - value = conversion_function(value) - # if functions arguments are present, expand the list to variables - # via the magic operator * - if function_parameters: - trans_val = callback_function(value, *function_parameters) - else: - trans_val = callback_function(value) - # append the transformed element to the list - return_list.append(trans_val) - return return_list - - -def _nested_alter( - document, - keys, - callback_function, - function_parameters, - conversion_function, - wild_alter, - in_place, - key_len, -): - - # return data if no callback_function is provided - if callback_function is None: - warnings.warn("Please provide a callback_function to nested_alter().") - return document - - # iterate over all given keys in the list - for key in keys: - # try to find the key: - findings = nested_lookup(key, document, with_keys=True, wild=wild_alter) - for k, v in findings.items(): - trans_val = _call_callback( - v, callback_function, function_parameters, conversion_function - ) - # use the transformed value and apply the update to the key - # (dont treat the lists as elements here) - document = nested_update( - document, k, trans_val, in_place=in_place, treat_as_element=False - ) - - return document diff --git a/config_file/nested-lookup/nested_lookup/nested_lookup.py b/config_file/nested-lookup/nested_lookup/nested_lookup.py deleted file mode 100644 index 0b11b4d..0000000 --- a/config_file/nested-lookup/nested_lookup/nested_lookup.py +++ /dev/null @@ -1,182 +0,0 @@ -from six import iteritems - -from collections import defaultdict - - -values_list = [] - - -def nested_lookup(key, document, wild=False, with_keys=False): - """Lookup a key in a nested document, return a list of values""" - if with_keys: - d = defaultdict(list) - for k, v in _nested_lookup(key, document, wild=wild, with_keys=with_keys): - d[k].append(v) - return d - return list(_nested_lookup(key, document, wild=wild, with_keys=with_keys)) - - -def _is_case_insensitive_substring(a, b): - """return True if `a` is a case insensitive substring of `b`, else False""" - return str(a).lower() in str(b).lower() - - -def _nested_lookup(key, document, wild=False, with_keys=False): - """Lookup a key in a nested document, yield a value""" - if isinstance(document, list): - for d in document: - for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): - yield result - - if isinstance(document, dict): - for k, v in iteritems(document): - if key == k or (wild and _is_case_insensitive_substring(key, k)): - if with_keys: - yield k, v - else: - yield v - if isinstance(v, dict): - for result in _nested_lookup(key, v, wild=wild, with_keys=with_keys): - yield result - elif isinstance(v, list): - for d in v: - for result in _nested_lookup( - key, d, wild=wild, with_keys=with_keys - ): - yield result - - -def get_all_keys(dictionary): - """ - Method to get all keys from a nested dictionary as a List - Args: - dictionary: Nested dictionary - Returns: - List of keys in the dictionary - """ - result_list = [] - - def recrusion(document): - if isinstance(document, list): - for list_items in document: - recrusion(document=list_items) - elif isinstance(document, dict): - for key, value in iteritems(document): - result_list.append(key) - recrusion(document=value) - return - - recrusion(document=dictionary) - return result_list - - -def get_occurrence_of_key(dictionary, key): - """ - Method to get occurrence of a key in a nested dictionary - - Args: - dictionary: Nested dictionary - key: Key to search for the occurrences - Return: - Number of occurrence (Integer) - """ - return _get_occurrence(dictionary=dictionary, item="key", keyword=key) - - -def get_occurrences_and_values(items, value): - """ - Method to get occurrence of a value in a nested list of dictionary - - Args: - items: list of dictionary: Nested dictionary - value: Value to search for the occurrences - - Return: - Dict where the key is the value arg and his value is a new - dict with occurrences and values - """ - occurrences = {} - occurrence = 0 - value_list = [] - - for item in items: - occurrence_result, values = _get_occurrence_with_values(dictionary=item, item="value", keyword=value) - occurrence = occurrence + occurrence_result - if occurrence_result: - value_list.extend(values) - - occurrences[value] = { - 'occurrences': occurrence, - 'values': value_list - } - - return occurrences - - -def _get_occurrence_with_values(dictionary, item, keyword): - occurrence = [0] - - result_recursion = _recursion(dictionary, item, keyword, occurrence, True) - - global values_list - values_list = [] - - return occurrence[0], result_recursion - - -def get_occurrence_of_value(dictionary, value): - """ - Method to get occurrence of a value in a nested dictionary - - Args: - dictionary: Nested dictionary - value: Value to search for the occurrences - Return: - Number of occurrence (Integer) - """ - return _get_occurrence(dictionary=dictionary, item="value", keyword=value) - - -def _recursion(dictionary, item, keyword, occurrence, with_values=False): - - global values_list - - if item == "key": - if dictionary.get(keyword) is not None: - occurrence[0] += 1 - elif keyword in list(dictionary.values()): - occurrence[0] += list(dictionary.values()).count(keyword) - if with_values: - values_list.append(dictionary) - for key, value in iteritems(dictionary): - if isinstance(value, dict): - _recursion(value, item, keyword, occurrence, with_values) - elif isinstance(value, list): - for list_items in value: - if hasattr(list_items, "items"): - _recursion(list_items, item, keyword, occurrence, with_values) - elif list_items == keyword: - occurrence[0] += 1 if item == "value" else 0 - - if values_list: - return values_list - - -def _get_occurrence(dictionary, item, keyword): - """ - Method to get occurrence of a key or value in a nested dictionary - - Args: - dictionary: Nested dictionary - item: Mostly (key or value) - keyword: key word to find occurrence - Return: - Number of occurrence of the given keyword in the dict - """ - occurrence = [0] - _recursion(dictionary, item, keyword, occurrence) - - global values_list - values_list = [] - - return occurrence[0] diff --git a/config_file/nested_lookup/.gitignore b/config_file/nested_lookup/.gitignore deleted file mode 100644 index c9bd7c4..0000000 --- a/config_file/nested_lookup/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -__pycache__* -.pytest_cache* -*.pyc -.coverage -build/* -dist/* -nested_lookup.egg-info/* -.cache -.vscode -git_commit.sh -venv diff --git a/config_file/nested_lookup/.travis.yml b/config_file/nested_lookup/.travis.yml deleted file mode 100644 index 5551955..0000000 --- a/config_file/nested_lookup/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -language: python -python: - - "2.7" - - "3.6" -# command to install dependencies -install: - - pip install . -# command to run tests -script: pytest \ No newline at end of file diff --git a/config_file/nested_lookup/CONTRIBUTING.rst b/config_file/nested_lookup/CONTRIBUTING.rst deleted file mode 100644 index dde9f01..0000000 --- a/config_file/nested_lookup/CONTRIBUTING.rst +++ /dev/null @@ -1,7 +0,0 @@ -The ``nested-lookup`` library is public domain and open source. - -To contribute please make sure all tests pass. - -If you are adding new functionality please write a unit test. - -To avoid discussions regarding code formatting, please use `black `_ with default settings. diff --git a/config_file/nested_lookup/__init__.py b/config_file/nested_lookup/__init__.py index 0840df6..768facc 100644 --- a/config_file/nested_lookup/__init__.py +++ b/config_file/nested_lookup/__init__.py @@ -5,4 +5,8 @@ from .nested_lookup import ( get_occurrence_of_value, get_occurrences_and_values ) -from .lookup_api import nested_update, nested_delete, nested_alter +from .lookup_api import ( + nested_update, + nested_delete, + nested_alter +) diff --git a/config_file/nested_lookup/lookup_api.py b/config_file/nested_lookup/lookup_api.py index a0af63a..e103534 100644 --- a/config_file/nested_lookup/lookup_api.py +++ b/config_file/nested_lookup/lookup_api.py @@ -1,7 +1,7 @@ import copy import warnings -from six import iteritems -from nested_lookup import nested_lookup + +from .nested_lookup import nested_lookup def nested_delete(document, key, in_place=False): @@ -26,7 +26,7 @@ def _nested_delete(document, key): elif isinstance(document, dict): if document.get(key): del document[key] - for dict_key, dict_value in iteritems(document): + for dict_key, dict_value in document.items(): _nested_delete(document=dict_value, key=key) return document @@ -34,16 +34,21 @@ def _nested_delete(document, key): def nested_update(document, key, value, in_place=False, treat_as_element=True): """ Method to update a key->value pair in a nested document + Args: document: Might be List of Dicts (or) Dict of Lists (or) Dict of List of Dicts etc... + key: Key to update the value + value: Value to set + in_place (bool): True: modify the dict in place; False: create a deep copy of the dict and modify it Defaults to False - treat_list_element (bool): + + treat_as_element (bool): True: if a list is provided as "value", the function trys to match the list elements to the occurences of the key. If the key occures more often than the provided list has @@ -51,6 +56,7 @@ def nested_update(document, key, value, in_place=False, treat_as_element=True): False: the provided list is treated as one scalar value and will be set as value to every key that matches. Defaults to True (because of backwards portability of the package). + Return: Returns a document that has updated key, value pair. """ @@ -89,7 +95,7 @@ def _nested_update(document, key, value): document=list_items, key=key, value=value ) elif isinstance(document, dict): - for dict_key, dict_value in iteritems(document): + for dict_key, dict_value in document.items(): if dict_key == key: document[key] = value[0] if len(value) > 1: diff --git a/config_file/nested_lookup/nested_lookup.py b/config_file/nested_lookup/nested_lookup.py index 0b11b4d..9454a44 100644 --- a/config_file/nested_lookup/nested_lookup.py +++ b/config_file/nested_lookup/nested_lookup.py @@ -1,5 +1,3 @@ -from six import iteritems - from collections import defaultdict @@ -29,7 +27,7 @@ def _nested_lookup(key, document, wild=False, with_keys=False): yield result if isinstance(document, dict): - for k, v in iteritems(document): + for k, v in document.items(): if key == k or (wild and _is_case_insensitive_substring(key, k)): if with_keys: yield k, v @@ -61,7 +59,7 @@ def get_all_keys(dictionary): for list_items in document: recrusion(document=list_items) elif isinstance(document, dict): - for key, value in iteritems(document): + for key, value in document.items(): result_list.append(key) recrusion(document=value) return @@ -148,7 +146,7 @@ def _recursion(dictionary, item, keyword, occurrence, with_values=False): occurrence[0] += list(dictionary.values()).count(keyword) if with_values: values_list.append(dictionary) - for key, value in iteritems(dictionary): + for key, value in dictionary.items(): if isinstance(value, dict): _recursion(value, item, keyword, occurrence, with_values) elif isinstance(value, list): diff --git a/config_file/nested_lookup/nested_lookup/__init__.py b/config_file/nested_lookup/nested_lookup/__init__.py deleted file mode 100644 index 0840df6..0000000 --- a/config_file/nested_lookup/nested_lookup/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .nested_lookup import ( - nested_lookup, - get_all_keys, - get_occurrence_of_key, - get_occurrence_of_value, - get_occurrences_and_values -) -from .lookup_api import nested_update, nested_delete, nested_alter diff --git a/config_file/nested_lookup/nested_lookup/lookup_api.py b/config_file/nested_lookup/nested_lookup/lookup_api.py deleted file mode 100644 index a0af63a..0000000 --- a/config_file/nested_lookup/nested_lookup/lookup_api.py +++ /dev/null @@ -1,214 +0,0 @@ -import copy -import warnings -from six import iteritems -from nested_lookup import nested_lookup - - -def nested_delete(document, key, in_place=False): - if not in_place: - document = copy.deepcopy(document) - return _nested_delete(document=document, key=key) - - -def _nested_delete(document, key): - """ - Method to delete a key->value pair from a nested document - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key: Key to delete - Return: - Returns a document that includes everything but the given key - """ - if isinstance(document, list): - for list_items in document: - _nested_delete(document=list_items, key=key) - elif isinstance(document, dict): - if document.get(key): - del document[key] - for dict_key, dict_value in iteritems(document): - _nested_delete(document=dict_value, key=key) - return document - - -def nested_update(document, key, value, in_place=False, treat_as_element=True): - """ - Method to update a key->value pair in a nested document - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key: Key to update the value - value: Value to set - in_place (bool): - True: modify the dict in place; - False: create a deep copy of the dict and modify it - Defaults to False - treat_list_element (bool): - True: if a list is provided as "value", the function trys - to match the list elements to the occurences of the key. - If the key occures more often than the provided list has - elements, the first element gets recycled. - False: the provided list is treated as one scalar value and - will be set as value to every key that matches. - Defaults to True (because of backwards portability of the package). - Return: - Returns a document that has updated key, value pair. - """ - - # check if a list or scalar value is provided and create a list - # from the scalar value - # check the length of the list and provide it to _nested_update - if not treat_as_element and not isinstance(value, list): - raise Exception( - "The value must be a list when treat_as_element is False." - ) - elif treat_as_element: - value = [value] - - if not in_place: - document = copy.deepcopy(document) - return _nested_update(document=document, key=key, value=value) - - -def _nested_update(document, key, value): - """ - Method to update a key->value pair in a nested document. - If the number of passed values is less than the number of key matches - when scanning for updates, use last value as default. - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key (str): Key to update the value - value (list): value(s) which should be used for replacement purpouse - Return: - Returns a document that has updated key, value pair. - """ - if isinstance(document, list): - for list_items in document: - _nested_update( - document=list_items, key=key, value=value - ) - elif isinstance(document, dict): - for dict_key, dict_value in iteritems(document): - if dict_key == key: - document[key] = value[0] - if len(value) > 1: - value.pop(0) - _nested_update( - document=dict_value, key=key, value=value - ) - return document - - -def nested_alter( - document, - key, - callback_function=None, - function_parameters=None, - conversion_function=None, - wild_alter=False, - in_place=False, -): - """ - Method to alter all values of the occurences of the key "key". - The provided callback_function is used to alter the scalar values - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key: Key to update the value - callback_function :A callback function which alters a scalar value - HINT: You should be aware that not every element might be of - the same type, please check this in your function! - function_parameters (list): - If the callback_function has additional input arguments except - the scalar value, please specify those in this list. - conversion_function: A conversion function like str() which should be - applied to every found value before it is passed to the - "callback_function" - wild_alter: Find matching elements via wild-match by the given keys - and alter those. - HINT: Keep in mind that the wild-match might return unexpected types! - in_place (bool): - True: modify the dict in place; - False: create a deep copy of the dict and modify it - Defaults to False - Return: - Returns a document that has updated key, value pair. - """ - # check if a list or scalar value is provided and create a list from - # the scalar value - # check the length of the list and provide it to _nested_update - if isinstance(key, list): - key_len = len(key) - else: - key = [key] - key_len = len(key) - - if not in_place: - document = copy.deepcopy(document) - return _nested_alter( - document=document, - keys=key, - callback_function=callback_function, - function_parameters=function_parameters, - conversion_function=conversion_function, - wild_alter=wild_alter, - in_place=in_place, - key_len=key_len, - ) - - -def _call_callback( - value_list, callback_function, function_parameters, conversion_function -): - """ - internal helper to call the callback function - """ - return_list = [] - # loop over all values - for value in value_list: - # apply the conversion function - if conversion_function is not None: - value = conversion_function(value) - # if functions arguments are present, expand the list to variables - # via the magic operator * - if function_parameters: - trans_val = callback_function(value, *function_parameters) - else: - trans_val = callback_function(value) - # append the transformed element to the list - return_list.append(trans_val) - return return_list - - -def _nested_alter( - document, - keys, - callback_function, - function_parameters, - conversion_function, - wild_alter, - in_place, - key_len, -): - - # return data if no callback_function is provided - if callback_function is None: - warnings.warn("Please provide a callback_function to nested_alter().") - return document - - # iterate over all given keys in the list - for key in keys: - # try to find the key: - findings = nested_lookup(key, document, with_keys=True, wild=wild_alter) - for k, v in findings.items(): - trans_val = _call_callback( - v, callback_function, function_parameters, conversion_function - ) - # use the transformed value and apply the update to the key - # (dont treat the lists as elements here) - document = nested_update( - document, k, trans_val, in_place=in_place, treat_as_element=False - ) - - return document diff --git a/config_file/nested_lookup/nested_lookup/nested_lookup.py b/config_file/nested_lookup/nested_lookup/nested_lookup.py deleted file mode 100644 index 0b11b4d..0000000 --- a/config_file/nested_lookup/nested_lookup/nested_lookup.py +++ /dev/null @@ -1,182 +0,0 @@ -from six import iteritems - -from collections import defaultdict - - -values_list = [] - - -def nested_lookup(key, document, wild=False, with_keys=False): - """Lookup a key in a nested document, return a list of values""" - if with_keys: - d = defaultdict(list) - for k, v in _nested_lookup(key, document, wild=wild, with_keys=with_keys): - d[k].append(v) - return d - return list(_nested_lookup(key, document, wild=wild, with_keys=with_keys)) - - -def _is_case_insensitive_substring(a, b): - """return True if `a` is a case insensitive substring of `b`, else False""" - return str(a).lower() in str(b).lower() - - -def _nested_lookup(key, document, wild=False, with_keys=False): - """Lookup a key in a nested document, yield a value""" - if isinstance(document, list): - for d in document: - for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): - yield result - - if isinstance(document, dict): - for k, v in iteritems(document): - if key == k or (wild and _is_case_insensitive_substring(key, k)): - if with_keys: - yield k, v - else: - yield v - if isinstance(v, dict): - for result in _nested_lookup(key, v, wild=wild, with_keys=with_keys): - yield result - elif isinstance(v, list): - for d in v: - for result in _nested_lookup( - key, d, wild=wild, with_keys=with_keys - ): - yield result - - -def get_all_keys(dictionary): - """ - Method to get all keys from a nested dictionary as a List - Args: - dictionary: Nested dictionary - Returns: - List of keys in the dictionary - """ - result_list = [] - - def recrusion(document): - if isinstance(document, list): - for list_items in document: - recrusion(document=list_items) - elif isinstance(document, dict): - for key, value in iteritems(document): - result_list.append(key) - recrusion(document=value) - return - - recrusion(document=dictionary) - return result_list - - -def get_occurrence_of_key(dictionary, key): - """ - Method to get occurrence of a key in a nested dictionary - - Args: - dictionary: Nested dictionary - key: Key to search for the occurrences - Return: - Number of occurrence (Integer) - """ - return _get_occurrence(dictionary=dictionary, item="key", keyword=key) - - -def get_occurrences_and_values(items, value): - """ - Method to get occurrence of a value in a nested list of dictionary - - Args: - items: list of dictionary: Nested dictionary - value: Value to search for the occurrences - - Return: - Dict where the key is the value arg and his value is a new - dict with occurrences and values - """ - occurrences = {} - occurrence = 0 - value_list = [] - - for item in items: - occurrence_result, values = _get_occurrence_with_values(dictionary=item, item="value", keyword=value) - occurrence = occurrence + occurrence_result - if occurrence_result: - value_list.extend(values) - - occurrences[value] = { - 'occurrences': occurrence, - 'values': value_list - } - - return occurrences - - -def _get_occurrence_with_values(dictionary, item, keyword): - occurrence = [0] - - result_recursion = _recursion(dictionary, item, keyword, occurrence, True) - - global values_list - values_list = [] - - return occurrence[0], result_recursion - - -def get_occurrence_of_value(dictionary, value): - """ - Method to get occurrence of a value in a nested dictionary - - Args: - dictionary: Nested dictionary - value: Value to search for the occurrences - Return: - Number of occurrence (Integer) - """ - return _get_occurrence(dictionary=dictionary, item="value", keyword=value) - - -def _recursion(dictionary, item, keyword, occurrence, with_values=False): - - global values_list - - if item == "key": - if dictionary.get(keyword) is not None: - occurrence[0] += 1 - elif keyword in list(dictionary.values()): - occurrence[0] += list(dictionary.values()).count(keyword) - if with_values: - values_list.append(dictionary) - for key, value in iteritems(dictionary): - if isinstance(value, dict): - _recursion(value, item, keyword, occurrence, with_values) - elif isinstance(value, list): - for list_items in value: - if hasattr(list_items, "items"): - _recursion(list_items, item, keyword, occurrence, with_values) - elif list_items == keyword: - occurrence[0] += 1 if item == "value" else 0 - - if values_list: - return values_list - - -def _get_occurrence(dictionary, item, keyword): - """ - Method to get occurrence of a key or value in a nested dictionary - - Args: - dictionary: Nested dictionary - item: Mostly (key or value) - keyword: key word to find occurrence - Return: - Number of occurrence of the given keyword in the dict - """ - occurrence = [0] - _recursion(dictionary, item, keyword, occurrence) - - global values_list - values_list = [] - - return occurrence[0] diff --git a/config_file/parsers/json_parser.py b/config_file/parsers/json_parser.py index 584a2d0..13b84b5 100644 --- a/config_file/parsers/json_parser.py +++ b/config_file/parsers/json_parser.py @@ -1,6 +1,6 @@ import json -from nested_lookup import ( +from config_file.nested_lookup import ( get_occurrence_of_key, nested_delete, nested_lookup, @@ -8,8 +8,7 @@ from nested_lookup import ( ) from config_file.exceptions import ParsingError -from config_file.parsers.base_parser import BaseParser -from config_file.parsers.parse_value import parse_value +from config_file.parsers import BaseParser, parse_value from config_file.utils import split_on_dot @@ -70,15 +69,14 @@ class JsonParser(BaseParser): split_keys = split_on_dot(key) result = self.parsed_content - # Used so we can more precisely specify what key is causing the error. - key_for_error = None + key_causing_error = None try: for key in split_keys: - key_for_error = key + key_causing_error = key result = result[key] except TypeError: raise ParsingError( - f"Cannot get {key} because {key_for_error} is not subscriptable." + f"Cannot get {key} because {key_causing_error} is not subscriptable." ) else: result = self.parsed_content[key] diff --git a/config_file/parsers/toml_parser.py b/config_file/parsers/toml_parser.py index 64ab216..ad3e7f4 100644 --- a/config_file/parsers/toml_parser.py +++ b/config_file/parsers/toml_parser.py @@ -1,6 +1,105 @@ -from config_file.parsers import BaseParser - import toml +from config_file.nested_lookup import ( + get_occurrence_of_key, + nested_delete, + nested_lookup, + nested_update, +) +from config_file.exceptions import ParsingError +from config_file.parsers import BaseParser, parse_value +from config_file.utils import split_on_dot + + class TomlParser(BaseParser): - pass \ No newline at end of file + """The TomlParser allows use of toml files with ConfigFile.""" + + def __init__(self, file_contents: str): + """Reads in the file contents into the parser.""" + super().__init__(file_contents) + + def parse(self, file_contents: str): + try: + return toml.loads(file_contents) + except toml.TomlDecodeError as error: + raise ParsingError(error) + + def get(self, key: str, parse_types: bool = False, all: bool = False): + """ + Retrieve values using a dot syntax. + + :param key: The key to retrieve. e.g. 'foo.bar.boo' + + :param parse_types: Whether you'd like the type parsed into its native one. + + :param all: Specify whether you'd like to recursively receive + all values that match the given key. Returned as a list. + e.g. 'foo' would retrieve all values that have the key 'foo', anywhere + in the json. + + :return: the value of the given key + """ + if all: + result = nested_lookup(key, self.parsed_content) + elif "." in key: + split_keys = split_on_dot(key) + result = self.parsed_content + + # Used so we can more precisely specify what key is causing the error. + key_for_error = None + try: + for key in split_keys: + key_for_error = key + result = result[key] + except TypeError: + raise ParsingError( + f"Cannot get {key} because {key_for_error} is not subscriptable." + ) + else: + result = self.parsed_content[key] + + return parse_value(result) if parse_types else result + + def set(self, key: str, value, all: bool = False): + if all: + nested_update(self.parsed_content, key, value, in_place=True) + elif "." in key: + indexes = split_on_dot(key) + indexes_length = len(indexes) + + ref = self.parsed_content + for index, key in enumerate(indexes): + if index == indexes_length - 1: + ref[key] = value + break + + ref = ref[key] + else: + self.parsed_content[key] = value + + return True + + def delete(self, key, all: bool = False): + if all: + nested_delete(self.parsed_content, key, in_place=True) + elif "." in key: + indexes = split_on_dot(key) + indexes_length = len(indexes) + + ref = self.parsed_content + for index, key in enumerate(indexes): + if index == indexes_length - 1: + del ref[key] + break + + ref = ref[key] + else: + del self.parsed_content[key] + + return True + + def stringify(self) -> str: + return json.dumps(self.parsed_content, indent=4) + + def has(self, search_key: str) -> bool: + return get_occurrence_of_key(self.parsed_content, key=search_key) > 0 -- cgit v1.2.3 From c476c1ed7065910fddc15ccfe67442c81f777bbf Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sun, 19 Jul 2020 14:59:05 -0500 Subject: Remove zsh info and add extras sub header --- config_file/parsers/abstract_parser.py | 75 ++++++++++++++++++++++++++++++++++ config_file/parsers/base_parser.py | 75 ---------------------------------- 2 files changed, 75 insertions(+), 75 deletions(-) create mode 100644 config_file/parsers/abstract_parser.py delete mode 100644 config_file/parsers/base_parser.py (limited to 'config_file') diff --git a/config_file/parsers/abstract_parser.py b/config_file/parsers/abstract_parser.py new file mode 100644 index 0000000..da6e90a --- /dev/null +++ b/config_file/parsers/abstract_parser.py @@ -0,0 +1,75 @@ +from abc import ABC, abstractmethod +from typing import Any + + +class BaseParser(ABC): + """ + The abstract base parser for all other parsers to implement. + + All the keys are specified in a 'dot' syntax. So if you have + sections with subsections with a key that you're trying to get + the value of, you'd use 'section.subsection.key' as the key parameter. + """ + + @abstractmethod + def __init__(self, file_contents: str) -> None: + """ + All parsers will take in the file as a string, parse + them internally to their own representation that they + can work with, and are able to stringify them back out + when we need to save (write) them. + + It is up to the caller of the parser to read in the file to + the parser and save the file when they are done using stringify(). + """ + self.content = file_contents + self.parsed_content = self.parse(self.content) + + @abstractmethod + def get(self, key: str, parse_types: bool = False, all: bool = False) -> Any: + """ + Retrieve the value of a key in its native type. + This means the string 'true' will be parsed back as the + boolean True. + + If parse_types is set to False, all values will be returned + back as strings. + """ + raise NotImplementedError + + @abstractmethod + def set(self, key: str, value: Any) -> bool: + """Sets the value of a key.""" + raise NotImplementedError + + @abstractmethod + def delete(self, section_key: str) -> bool: + """Deletes a key/value pair or entire sections.""" + raise NotImplementedError + + @abstractmethod + def stringify(self) -> str: + """ + Using self.__contents, converts the parsed internal representation + of the file back into a string. + """ + raise NotImplementedError + + @abstractmethod + def has(self, section_key: str) -> bool: + """ + Check if a section, sub-section, or key exists in the file + using a section.key format. + + Some formats, like JSON, do not have sections and therefore, + it would only be checking if + """ + raise NotImplementedError + + @abstractmethod + def parse(self, file_contents: str) -> Any: + """ + Parse the file_contents into an internal representation + the given parser can work with. + """ + raise NotImplementedError diff --git a/config_file/parsers/base_parser.py b/config_file/parsers/base_parser.py deleted file mode 100644 index da6e90a..0000000 --- a/config_file/parsers/base_parser.py +++ /dev/null @@ -1,75 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any - - -class BaseParser(ABC): - """ - The abstract base parser for all other parsers to implement. - - All the keys are specified in a 'dot' syntax. So if you have - sections with subsections with a key that you're trying to get - the value of, you'd use 'section.subsection.key' as the key parameter. - """ - - @abstractmethod - def __init__(self, file_contents: str) -> None: - """ - All parsers will take in the file as a string, parse - them internally to their own representation that they - can work with, and are able to stringify them back out - when we need to save (write) them. - - It is up to the caller of the parser to read in the file to - the parser and save the file when they are done using stringify(). - """ - self.content = file_contents - self.parsed_content = self.parse(self.content) - - @abstractmethod - def get(self, key: str, parse_types: bool = False, all: bool = False) -> Any: - """ - Retrieve the value of a key in its native type. - This means the string 'true' will be parsed back as the - boolean True. - - If parse_types is set to False, all values will be returned - back as strings. - """ - raise NotImplementedError - - @abstractmethod - def set(self, key: str, value: Any) -> bool: - """Sets the value of a key.""" - raise NotImplementedError - - @abstractmethod - def delete(self, section_key: str) -> bool: - """Deletes a key/value pair or entire sections.""" - raise NotImplementedError - - @abstractmethod - def stringify(self) -> str: - """ - Using self.__contents, converts the parsed internal representation - of the file back into a string. - """ - raise NotImplementedError - - @abstractmethod - def has(self, section_key: str) -> bool: - """ - Check if a section, sub-section, or key exists in the file - using a section.key format. - - Some formats, like JSON, do not have sections and therefore, - it would only be checking if - """ - raise NotImplementedError - - @abstractmethod - def parse(self, file_contents: str) -> Any: - """ - Parse the file_contents into an internal representation - the given parser can work with. - """ - raise NotImplementedError -- cgit v1.2.3 From 461cb78f0a56b9293b5ade10fe90ec4aa77d23ad Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sun, 19 Jul 2020 15:02:13 -0500 Subject: Change name of BaseParser to AbstractParser --- config_file/__init__.py | 4 +- config_file/config_file.py | 12 ++-- config_file/parsers/__init__.py | 14 ++--- config_file/parsers/abstract_parser.py | 20 +++---- config_file/parsers/ini_parser.py | 4 +- config_file/parsers/json_parser.py | 10 ++-- config_file/parsers/toml_parser.py | 104 +-------------------------------- config_file/parsers/yaml_parser.py | 5 +- 8 files changed, 38 insertions(+), 135 deletions(-) (limited to 'config_file') diff --git a/config_file/__init__.py b/config_file/__init__.py index 1b2094f..3c43d33 100644 --- a/config_file/__init__.py +++ b/config_file/__init__.py @@ -1,6 +1,6 @@ from config_file.config_file import ConfigFile from config_file.exceptions import ConfigFileError, ParsingError -from config_file.parsers import BaseParser +from config_file.parsers import AbstractParser __version__ = "0.10.0" @@ -8,5 +8,5 @@ __all__ = [ "ConfigFile", "ParsingError", "ConfigFileError", - "BaseParser", + "AbstractParser", ] diff --git a/config_file/config_file.py b/config_file/config_file.py index de8b851..34df561 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -4,7 +4,7 @@ from shutil import copyfile from typing import Any, Type, Union from config_file.exceptions import ParsingError -from config_file.parsers import BaseParser, IniParser, JsonParser +from config_file.parsers import AbstractParser, IniParser, JsonParser from config_file.utils import create_config_path, read_file, split_on_dot @@ -12,7 +12,7 @@ class ConfigFile: def __init__( self, file_path: Type[Union[str, Type[PurePath]]], - parser: Type[BaseParser] = None, + parser: Type[AbstractParser] = None, ) -> None: """ Saves the config file path and expands it if needed, reads in @@ -37,8 +37,8 @@ class ConfigFile: return self.__path def __determine_parser( - self, file_path: Type[PurePath], parser: Type[BaseParser] = None - ) -> Type[BaseParser]: + self, file_path: Type[PurePath], parser: Type[AbstractParser] = None + ) -> Type[AbstractParser]: if parser is not None and not inspect.isabstract(parser): return parser(self.__contents) @@ -108,7 +108,9 @@ class ConfigFile: """ return self.__parser.has(section_key) - def restore_original(self, original_file_path: Union[str, Type[PurePath]] = None) -> bool: + def restore_original( + self, original_file_path: Union[str, Type[PurePath]] = None + ) -> bool: """ Restores the original the config file by deleting it and copying the original back in its place. The internal contents of this config file object are then set diff --git a/config_file/parsers/__init__.py b/config_file/parsers/__init__.py index 8da520e..554b624 100644 --- a/config_file/parsers/__init__.py +++ b/config_file/parsers/__init__.py @@ -1,17 +1,17 @@ -from .base_parser import BaseParser +from .abstract_parser import AbstractParser from .ini_parser import IniParser from .json_parser import JsonParser -from .toml_parser import TomlParser -from .yaml_parser import YamlParser from .parse_value import ( - parse_value, can_be_parsed_as_bool, can_be_parsed_as_float, - can_be_parsed_as_int + can_be_parsed_as_int, + parse_value, ) +from .toml_parser import TomlParser +from .yaml_parser import YamlParser __all__ = [ - "BaseParser", + "AbstractParser", "IniParser", "JsonParser", "TomlParser", @@ -19,5 +19,5 @@ __all__ = [ "parse_value", "can_be_parsed_as_bool", "can_be_parsed_as_float", - "can_be_parsed_as_int" + "can_be_parsed_as_int", ] diff --git a/config_file/parsers/abstract_parser.py b/config_file/parsers/abstract_parser.py index da6e90a..8f1bee1 100644 --- a/config_file/parsers/abstract_parser.py +++ b/config_file/parsers/abstract_parser.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from typing import Any -class BaseParser(ABC): +class AbstractParser(ABC): """ The abstract base parser for all other parsers to implement. @@ -25,6 +25,14 @@ class BaseParser(ABC): self.content = file_contents self.parsed_content = self.parse(self.content) + @abstractmethod + def parse(self, file_contents: str) -> Any: + """ + Parse the file_contents into an internal representation + the given parser can work with. + """ + raise NotImplementedError + @abstractmethod def get(self, key: str, parse_types: bool = False, all: bool = False) -> Any: """ @@ -62,14 +70,6 @@ class BaseParser(ABC): using a section.key format. Some formats, like JSON, do not have sections and therefore, - it would only be checking if - """ - raise NotImplementedError - - @abstractmethod - def parse(self, file_contents: str) -> Any: - """ - Parse the file_contents into an internal representation - the given parser can work with. + it would only be checking if a certain key exists. """ raise NotImplementedError diff --git a/config_file/parsers/ini_parser.py b/config_file/parsers/ini_parser.py index 2db1ae8..164ff6d 100644 --- a/config_file/parsers/ini_parser.py +++ b/config_file/parsers/ini_parser.py @@ -2,12 +2,12 @@ import configparser from io import StringIO from config_file.exceptions import ParsingError -from config_file.parsers.base_parser import BaseParser +from config_file.parsers.abstract_parser import AbstractParser from config_file.parsers.parse_value import parse_value from config_file.utils import split_on_dot -class IniParser(BaseParser): +class IniParser(AbstractParser): def __init__(self, file_contents: str): """Reads in the file contents into the configparser.""" super().__init__(file_contents) diff --git a/config_file/parsers/json_parser.py b/config_file/parsers/json_parser.py index 13b84b5..237c0b5 100644 --- a/config_file/parsers/json_parser.py +++ b/config_file/parsers/json_parser.py @@ -1,18 +1,17 @@ import json +from config_file.exceptions import ParsingError from config_file.nested_lookup import ( get_occurrence_of_key, nested_delete, nested_lookup, nested_update, ) - -from config_file.exceptions import ParsingError -from config_file.parsers import BaseParser, parse_value +from config_file.parsers import AbstractParser, parse_value from config_file.utils import split_on_dot -class JsonParser(BaseParser): +class JsonParser(AbstractParser): """ The JsonParser allows use of json files with ConfigFile. The syntax it supports is a bit more flexible than the IniParser. @@ -76,7 +75,8 @@ class JsonParser(BaseParser): result = result[key] except TypeError: raise ParsingError( - f"Cannot get {key} because {key_causing_error} is not subscriptable." + f"Cannot get {key} because {key_causing_error} " + "is not subscriptable." ) else: result = self.parsed_content[key] diff --git a/config_file/parsers/toml_parser.py b/config_file/parsers/toml_parser.py index ad3e7f4..4905aec 100644 --- a/config_file/parsers/toml_parser.py +++ b/config_file/parsers/toml_parser.py @@ -1,105 +1,5 @@ -import toml +from config_file.parsers import AbstractParser -from config_file.nested_lookup import ( - get_occurrence_of_key, - nested_delete, - nested_lookup, - nested_update, -) -from config_file.exceptions import ParsingError -from config_file.parsers import BaseParser, parse_value -from config_file.utils import split_on_dot - -class TomlParser(BaseParser): +class TomlParser(AbstractParser): """The TomlParser allows use of toml files with ConfigFile.""" - - def __init__(self, file_contents: str): - """Reads in the file contents into the parser.""" - super().__init__(file_contents) - - def parse(self, file_contents: str): - try: - return toml.loads(file_contents) - except toml.TomlDecodeError as error: - raise ParsingError(error) - - def get(self, key: str, parse_types: bool = False, all: bool = False): - """ - Retrieve values using a dot syntax. - - :param key: The key to retrieve. e.g. 'foo.bar.boo' - - :param parse_types: Whether you'd like the type parsed into its native one. - - :param all: Specify whether you'd like to recursively receive - all values that match the given key. Returned as a list. - e.g. 'foo' would retrieve all values that have the key 'foo', anywhere - in the json. - - :return: the value of the given key - """ - if all: - result = nested_lookup(key, self.parsed_content) - elif "." in key: - split_keys = split_on_dot(key) - result = self.parsed_content - - # Used so we can more precisely specify what key is causing the error. - key_for_error = None - try: - for key in split_keys: - key_for_error = key - result = result[key] - except TypeError: - raise ParsingError( - f"Cannot get {key} because {key_for_error} is not subscriptable." - ) - else: - result = self.parsed_content[key] - - return parse_value(result) if parse_types else result - - def set(self, key: str, value, all: bool = False): - if all: - nested_update(self.parsed_content, key, value, in_place=True) - elif "." in key: - indexes = split_on_dot(key) - indexes_length = len(indexes) - - ref = self.parsed_content - for index, key in enumerate(indexes): - if index == indexes_length - 1: - ref[key] = value - break - - ref = ref[key] - else: - self.parsed_content[key] = value - - return True - - def delete(self, key, all: bool = False): - if all: - nested_delete(self.parsed_content, key, in_place=True) - elif "." in key: - indexes = split_on_dot(key) - indexes_length = len(indexes) - - ref = self.parsed_content - for index, key in enumerate(indexes): - if index == indexes_length - 1: - del ref[key] - break - - ref = ref[key] - else: - del self.parsed_content[key] - - return True - - def stringify(self) -> str: - return json.dumps(self.parsed_content, indent=4) - - def has(self, search_key: str) -> bool: - return get_occurrence_of_key(self.parsed_content, key=search_key) > 0 diff --git a/config_file/parsers/yaml_parser.py b/config_file/parsers/yaml_parser.py index 48722b5..a45d7f6 100644 --- a/config_file/parsers/yaml_parser.py +++ b/config_file/parsers/yaml_parser.py @@ -1,4 +1,5 @@ -from config_file.parsers import BaseParser +from config_file.parsers import AbstractParser -class YamlParser(BaseParser): + +class YamlParser(AbstractParser): pass -- cgit v1.2.3 From 6eab76b24dfc831b8c41c445d98175c3276b20a5 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sun, 19 Jul 2020 20:28:40 -0500 Subject: Add UnrecognizedFileError and MissingDependencyError --- config_file/__init__.py | 13 ++++++++++--- config_file/exceptions.py | 13 +++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) (limited to 'config_file') diff --git a/config_file/__init__.py b/config_file/__init__.py index 3c43d33..668c0a9 100644 --- a/config_file/__init__.py +++ b/config_file/__init__.py @@ -1,12 +1,19 @@ from config_file.config_file import ConfigFile -from config_file.exceptions import ConfigFileError, ParsingError +from config_file.exceptions import ( + ConfigFileError, + MissingDependencyError, + ParsingError, + UnrecognizedFileError, +) from config_file.parsers import AbstractParser __version__ = "0.10.0" __all__ = [ "ConfigFile", - "ParsingError", - "ConfigFileError", "AbstractParser", + "ConfigFileError", + "ParsingError", + "UnrecognizedFileError", + "MissingDependencyError", ] diff --git a/config_file/exceptions.py b/config_file/exceptions.py index e1af755..03ef052 100644 --- a/config_file/exceptions.py +++ b/config_file/exceptions.py @@ -4,3 +4,16 @@ class ConfigFileError(Exception): class ParsingError(ConfigFileError): """Unable to parse the configuration file.""" + + +class MissingDependencyError(ConfigFileError): + """ + PyYaml or toml is not installed, but the parsers + for them were attempted to be used. + """ + + +class UnrecognizedFileError(ConfigFileError): + """ + A missing file extension or a file type not supported. + """ -- cgit v1.2.3 From bc05c79f915e2b45b7f13f143b4c01f4c49013f8 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Mon, 20 Jul 2020 21:59:22 -0500 Subject: Add FLOAT_REGEX constant --- config_file/parsers/parse_value.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/parse_value.py b/config_file/parsers/parse_value.py index e58781a..26bdf2e 100644 --- a/config_file/parsers/parse_value.py +++ b/config_file/parsers/parse_value.py @@ -1,5 +1,5 @@ """ -Parse types allows you to check or parse a string into its native type. +This module allows you to parse a string into its native type. Example: parse_value('5') -> 5 (int) @@ -17,6 +17,7 @@ Example: "5".isdigit() -> True "-5".isdigit() -> False can_be_parsed_as_int("-5") -> True +parse_value("-5") -> -5 """ import re from distutils.util import strtobool @@ -65,6 +66,8 @@ def can_be_parsed_as_int(value) -> bool: def can_be_parsed_as_float(value) -> bool: + FLOAT_REGEX = r"^\d*\.\d+$" + if type(value) is float: return True @@ -72,9 +75,9 @@ def can_be_parsed_as_float(value) -> bool: return False if value.startswith("-"): - return bool(re.match(r"^\d*\.\d+$", value[1:])) + return bool(re.match(FLOAT_REGEX, value[1:])) - return bool(re.match(r"^\d*\.\d+$", value)) + return bool(re.match(FLOAT_REGEX, value)) def can_be_parsed_as_bool(value) -> bool: -- cgit v1.2.3 From 8c28da679092d9debf7140974fa7d28dfeda8e21 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Mon, 20 Jul 2020 22:49:58 -0500 Subject: Redo docstrings and api for abstract parser --- config_file/parsers/abstract_parser.py | 122 ++++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 40 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/abstract_parser.py b/config_file/parsers/abstract_parser.py index 8f1bee1..ed9430b 100644 --- a/config_file/parsers/abstract_parser.py +++ b/config_file/parsers/abstract_parser.py @@ -3,73 +3,115 @@ from typing import Any class AbstractParser(ABC): - """ - The abstract base parser for all other parsers to implement. + """Abstract Parser is the API contract for all parsers. + + Every parser in ConfigFile implements this parser, and + any user created parsers must implement this parser in + order for this package to be able to use it. + + Every key is specified in a dot (.) syntax + e.g. retrieving.some.deeply.nested.value + + If there is no dot in the key, it is assumed we are + performing an action on a top level key, which could + be an entire section. - All the keys are specified in a 'dot' syntax. So if you have - sections with subsections with a key that you're trying to get - the value of, you'd use 'section.subsection.key' as the key parameter. + Args: + file_contents: The stringified version of a file. + + Attributes: + file_contents: The stringified version of a file. """ @abstractmethod def __init__(self, file_contents: str) -> None: - """ - All parsers will take in the file as a string, parse - them internally to their own representation that they - can work with, and are able to stringify them back out - when we need to save (write) them. - - It is up to the caller of the parser to read in the file to - the parser and save the file when they are done using stringify(). - """ - self.content = file_contents - self.parsed_content = self.parse(self.content) + self.file_contents = file_contents @abstractmethod - def parse(self, file_contents: str) -> Any: - """ - Parse the file_contents into an internal representation - the given parser can work with. + def reset_internal_contents(self, file_contents: str) -> None: + """Reset the state of this parser to a new file. + + When restoring to the original file in ConfigFile, + we need a way to reset the internal contents to this + new file. + + While this may be as simple as setting the file_contents + attribute value to the passed in file_contents argument, + it likely would not be. This is because the initial + passed in file contents would likely be parsed into some + special form for the parser to work with more easily. + + Args: + file_contents: The file contents of the new file. """ raise NotImplementedError @abstractmethod - def get(self, key: str, parse_types: bool = False, all: bool = False) -> Any: - """ - Retrieve the value of a key in its native type. - This means the string 'true' will be parsed back as the - boolean True. + def get(self, key: str) -> Any: + """Retrieve the value of a key from the working file. + + Args: + key: The key to retrieve. - If parse_types is set to False, all values will be returned - back as strings. + Returns: + Whatever value the given key happens to contain. + + Raises: + MissingKeyError: If the key to retrieve does not exist. """ raise NotImplementedError @abstractmethod - def set(self, key: str, value: Any) -> bool: - """Sets the value of a key.""" + def set(self, key: str, value: Any) -> None: + """Set the value of a key in the working file. + + If the key does not exist, it should be created. + + Args: + key: The key to set. + value: The value to set the key to. + """ raise NotImplementedError @abstractmethod - def delete(self, section_key: str) -> bool: - """Deletes a key/value pair or entire sections.""" + def delete(self, key: str) -> None: + """Deletes a key from the working file. + + Args: + key: The key we would like to remove. + + Raises: + MissingKeyError: If the key to remove does not exist. + """ raise NotImplementedError @abstractmethod def stringify(self) -> str: - """ - Using self.__contents, converts the parsed internal representation - of the file back into a string. + """Stringify the working file. + + Since the ConfigFile does not write out to disk + after every operation, this method is what is + used as the output for saving the file again. + This parser should never write out to the disk. + + Returns: + The file we are working with as a string. """ raise NotImplementedError @abstractmethod - def has(self, section_key: str) -> bool: - """ - Check if a section, sub-section, or key exists in the file - using a section.key format. + def has(self, key: str) -> bool: + """Check whether the given key is in the parsed working file. + + This should not behave like a wild card, as in check + if the key exists anywhere in the working file. Instead, + if a key is given without a dot, it should be assumed we + are checking the top level. + + Args: + key: The key to search for. - Some formats, like JSON, do not have sections and therefore, - it would only be checking if a certain key exists. + Returns: + True if the working file has the key. False otherwise. """ raise NotImplementedError -- cgit v1.2.3 From ce71e05c2615350dba997b671ee63f8bd607cf03 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Mon, 20 Jul 2020 22:50:39 -0500 Subject: Remove parser imports from __init__ --- config_file/parsers/__init__.py | 23 ----------------------- 1 file changed, 23 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/__init__.py b/config_file/parsers/__init__.py index 554b624..e69de29 100644 --- a/config_file/parsers/__init__.py +++ b/config_file/parsers/__init__.py @@ -1,23 +0,0 @@ -from .abstract_parser import AbstractParser -from .ini_parser import IniParser -from .json_parser import JsonParser -from .parse_value import ( - can_be_parsed_as_bool, - can_be_parsed_as_float, - can_be_parsed_as_int, - parse_value, -) -from .toml_parser import TomlParser -from .yaml_parser import YamlParser - -__all__ = [ - "AbstractParser", - "IniParser", - "JsonParser", - "TomlParser", - "YamlParser", - "parse_value", - "can_be_parsed_as_bool", - "can_be_parsed_as_float", - "can_be_parsed_as_int", -] -- cgit v1.2.3 From ba5d3e2541670829697d794943322dd5f2a453af Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Mon, 20 Jul 2020 22:51:14 -0500 Subject: Add toml and yaml parsers --- config_file/parsers/toml_parser.py | 29 ++++++++++++++++++++++++++--- config_file/parsers/yaml_parser.py | 30 +++++++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 6 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/toml_parser.py b/config_file/parsers/toml_parser.py index 4905aec..d369003 100644 --- a/config_file/parsers/toml_parser.py +++ b/config_file/parsers/toml_parser.py @@ -1,5 +1,28 @@ -from config_file.parsers import AbstractParser +try: + import tomlkit + TOML_AVAILABLE = True +except ImportError: + TOML_AVAILABLE = False -class TomlParser(AbstractParser): - """The TomlParser allows use of toml files with ConfigFile.""" +from config_file.exceptions import MissingDependencyError + +from .base_parser import BaseParser + + +class TomlParser(BaseParser): + def __init__(self, file_contents: str): + if not TOML_AVAILABLE: + raise MissingDependencyError( + "It doesn't appear `toml` is installed, but a toml " + "file was attempted to be used. Install the `toml` " + "extra first with `pip install config-file[toml]`." + ) + + super().__init__( + file_contents=file_contents, + module=tomlkit, + # tomlkit doesn't expose it's exceptions, but + # it's parsing error is a subclass of ValueError + decode_error=ValueError, + ) diff --git a/config_file/parsers/yaml_parser.py b/config_file/parsers/yaml_parser.py index a45d7f6..77b59fe 100644 --- a/config_file/parsers/yaml_parser.py +++ b/config_file/parsers/yaml_parser.py @@ -1,5 +1,29 @@ -from config_file.parsers import AbstractParser +try: + import yaml + YAML_AVAILABLE = True -class YamlParser(AbstractParser): - pass + # Conform yaml to json and + # toml package syntax + yaml.loads = yaml.load + yaml.dumps = yaml.dump +except ImportError: + YAML_AVAILABLE = False + +from config_file.exceptions import MissingDependencyError + +from .base_parser import BaseParser + + +class YamlParser(BaseParser): + def __init__(self, file_contents: str): + if not YAML_AVAILABLE: + raise MissingDependencyError( + "It doesn't appear `PyYaml` is installed, but a yaml " + "file was attempted to be used. Install the `toml` " + "extra first with `pip install config-file[toml]`." + ) + + super().__init__( + file_contents=file_contents, module=yaml, decode_error=yaml.YAMLError + ) -- cgit v1.2.3 From 65d4715b0b2b9974d95ac36271caa8e177bc98fa Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Mon, 20 Jul 2020 22:51:51 -0500 Subject: Add BaseParser for all other parsers This allows one implementation for all parsers, with the parsers overriding it for a custom one if needed for that parser. --- config_file/parsers/base_parser.py | 86 ++++++++++++++++++++++ config_file/parsers/ini_parser.py | 145 ++++++++++--------------------------- config_file/parsers/json_parser.py | 127 ++------------------------------ 3 files changed, 130 insertions(+), 228 deletions(-) create mode 100644 config_file/parsers/base_parser.py (limited to 'config_file') diff --git a/config_file/parsers/base_parser.py b/config_file/parsers/base_parser.py new file mode 100644 index 0000000..09fbbbf --- /dev/null +++ b/config_file/parsers/base_parser.py @@ -0,0 +1,86 @@ +from typing import Any + +from config_file.exceptions import MissingKeyError, ParsingError +from config_file.nested_lookup import get_occurrence_of_key +from config_file.utils import split_on_dot + +from .abstract_parser import AbstractParser + + +class BaseParser(AbstractParser): + def __init__(self, file_contents: str, module, decode_error): + super().__init__(file_contents) + self.__module = module + self.__decode_error = decode_error + self.parsed_content = self.parse_file_contents(module, decode_error) + + def parse_file_contents(self, module, decode_error) -> Any: + try: + return module.loads(self.file_contents) + except decode_error as error: + raise ParsingError(error) + + def reset_internal_contents(self, file_contents: str) -> None: + self.file_contents = file_contents + self.parsed_content = self.parse_file_contents( + self.__module, self.__decode_error + ) + + def get(self, search_key: str) -> Any: + if "." not in search_key: + result = self.parsed_content[search_key] + return result + else: + split_keys = split_on_dot(search_key) + + result = self.parsed_content + key_causing_error = None + try: + for key in split_keys: + key_causing_error = key + result = result[key] + except TypeError: + raise ParsingError( + f"Cannot get {search_key} because " + f"{key_causing_error} is not subscriptable." + ) + + return result + + def set(self, key: str, value: Any) -> None: + if "." not in key: + self.parsed_content[key] = value + return + + keys = split_on_dot(key) + content_reference = self.parsed_content + for index, key in enumerate(keys): + if index == len(keys) - 1: + content_reference[key] = value + return + + content_reference = content_reference[key] + + def delete(self, key) -> None: + try: + if "." not in key: + del self.parsed_content[key] + else: + indexes = split_on_dot(key) + indexes_length = len(indexes) + + ref = self.parsed_content + for index, key in enumerate(indexes): + if index == indexes_length - 1: + del ref[key] + break + + ref = ref[key] + except KeyError: + raise MissingKeyError(f"The specified key '{key}' was not found.") + + def stringify(self) -> str: + return self.__module.dumps(self.parsed_content) + + def has(self, search_key: str) -> bool: + return get_occurrence_of_key(self.parsed_content, key=search_key) > 0 diff --git a/config_file/parsers/ini_parser.py b/config_file/parsers/ini_parser.py index 164ff6d..2a29ba7 100644 --- a/config_file/parsers/ini_parser.py +++ b/config_file/parsers/ini_parser.py @@ -1,124 +1,53 @@ import configparser from io import StringIO -from config_file.exceptions import ParsingError -from config_file.parsers.abstract_parser import AbstractParser -from config_file.parsers.parse_value import parse_value -from config_file.utils import split_on_dot +from .base_parser import BaseParser -class IniParser(AbstractParser): - def __init__(self, file_contents: str): - """Reads in the file contents into the configparser.""" - super().__init__(file_contents) - - def parse(self, file_contents: str): - try: - parser = configparser.ConfigParser() - parser.read_string(file_contents) - return parser - except configparser.Error as error: - raise ParsingError(error) - - def get(self, section_key: str, parse_types: bool = False): - """ - Read the value of `section.key` of the config file. - - :param section_key: The section and key to read from in the config file. - e.g. 'section1.key2' - :param parse_types: Coerces the return value to its native type. - :return: The value of the key, parsed to its native type if parse_types is True. - - :raises ValueError: if the section_key given is not in a dot format. e.g. - 'section1.key' - :raises ParsingError: if the specified `section.key` is not found, in an - invalid format, or if we are unable to coerce the return value to value_type. - """ - if "." not in section_key: - return self.__retrieve_section(section_key, parse_types) - - try: - section, key = split_on_dot(section_key, only_last_dot=True) - value = self.parsed_content.get(section, key) - return parse_value(value) if parse_types else value - except configparser.Error as error: - raise ParsingError(error.message) - - def __retrieve_section(self, section, parse_types): - items = dict(self.parsed_content.items(section)) - - if not parse_types: - return items - - for item in items: - items[item] = parse_value(items[item]) - - return items - - def set(self, section_key: str, value) -> bool: - """ - Sets the value of 'section.key' of the config file. If the specified section - is not in the configuration file, it will be created before adding the key - to it. +class IniParserCompatibility: + """ + Provides compatibility with the json, yaml, + and toml API so we can use the BaseParser + easily, while still being able to leverage + the built-in configparser to parse the file. - :param section_key: The key to set from the config file. e.g. 'section1.key' + It is used as the `module` parameter + for BaseParser, which requires us to have a + `dumps` and `loads` method. + """ - :param value: The value to set the key to. It can be any type that can - be converted to a string. + @staticmethod + def loads(file_contents: str) -> dict: + parser = configparser.ConfigParser() + parser.read_string(file_contents) - :return: True if the setting was successful. + return IniParserCompatibility.__create_configparser_dict(parser) - :raises ValueError: If there is no dot (.) in section_key - """ - section, key = split_on_dot(section_key, only_last_dot=True) - - if value is not None and not self.parsed_content.has_section(section): - self.parsed_content.add_section(section) - - if not isinstance(value, str): - value = str(value) - - self.parsed_content.set(section, key, value) - return True - - def delete(self, section_key: str) -> bool: - """ - Deletes a key or an entire section. - - :param section_key: The key to delete from the config file. - e.g. 'ocr.engine'. If no dot (.) is present, it will assume you are trying - to delete the entire section. - - :return: True if the deletion succeeded. + @staticmethod + def dumps(file_contents: dict) -> str: + parser = configparser.ConfigParser() + buffer = StringIO() - :raise ValueError: If the section or key does not exist in the config file. - """ - if "." not in section_key: - if not self.parsed_content.has_section(section_key): - raise ValueError( - f"Cannot delete {section_key} because it is not in the config file." - ) + parser.read_dict(file_contents) + parser.write(buffer) - self.parsed_content.remove_section(section_key) - return True + return buffer.getvalue() - section, key = split_on_dot(section_key, only_last_dot=True) - if key not in self.parsed_content[section]: - raise ValueError( - f"Cannot delete {section}.{key} because {key} is not in {section}." - ) + @staticmethod + def __create_configparser_dict(parser: configparser.ConfigParser) -> dict: + result = {} + items = dict(parser.items()) - self.parsed_content.remove_option(section, key) - return True + for section in parser.sections(): + result.update({str(section): dict(items[section])}) - def stringify(self) -> str: - buffer = StringIO() - self.parsed_content.write(buffer) - return buffer.getvalue() + return result - def has(self, section_key: str) -> bool: - if "." not in section_key: - return self.parsed_content.has_section(section_key) - section, key = split_on_dot(section_key, only_last_dot=True) - return self.parsed_content.has_option(section, key) +class IniParser(BaseParser): + def __init__(self, file_contents: str): + super().__init__( + file_contents=file_contents, + module=IniParserCompatibility, + decode_error=configparser.Error, + ) diff --git a/config_file/parsers/json_parser.py b/config_file/parsers/json_parser.py index 237c0b5..5314cd7 100644 --- a/config_file/parsers/json_parser.py +++ b/config_file/parsers/json_parser.py @@ -1,128 +1,15 @@ import json -from config_file.exceptions import ParsingError -from config_file.nested_lookup import ( - get_occurrence_of_key, - nested_delete, - nested_lookup, - nested_update, -) -from config_file.parsers import AbstractParser, parse_value -from config_file.utils import split_on_dot +from .base_parser import BaseParser -class JsonParser(AbstractParser): - """ - The JsonParser allows use of json files with ConfigFile. The syntax it - supports is a bit more flexible than the IniParser. - - By default, the dot syntax is used. However, if you'd like to get/set/delete all - of a particular key, you can specify so with an optional parameter. - - Example: - - json = { - "key1": { - "key2": "foo", - "key1": "bar" - }, - "foo": "bar" - } - parser = JsonParser(json.dumps(json)) - parser.get('key1') - >>> { "key2": "foo", "key1": "bar" } - parser.get('key1.key2') - >>> "foo" - parser.get('key1', all=True) - >>> [{ "key2": "foo", "key1": "bar" }, "bar"] - """ - +class JsonParser(BaseParser): def __init__(self, file_contents: str): - """Reads in the file contents into the parser.""" - super().__init__(file_contents) - - def parse(self, file_contents: str): - try: - return json.loads(file_contents) - except json.decoder.JSONDecodeError as error: - raise ParsingError(error) - - def get(self, key: str, parse_types: bool = False, all: bool = False): - """ - Retrieve values using a dot syntax. - - :param key: The key to retrieve. e.g. 'foo.bar.boo' - - :param parse_types: Whether you'd like the type parsed into its native one. - - :param all: Specify whether you'd like to recursively receive - all values that match the given key. Returned as a list. - e.g. 'foo' would retrieve all values that have the key 'foo', anywhere - in the json. - - :return: the value of the given key - """ - if all: - result = nested_lookup(key, self.parsed_content) - elif "." in key: - split_keys = split_on_dot(key) - result = self.parsed_content - - key_causing_error = None - try: - for key in split_keys: - key_causing_error = key - result = result[key] - except TypeError: - raise ParsingError( - f"Cannot get {key} because {key_causing_error} " - "is not subscriptable." - ) - else: - result = self.parsed_content[key] - - return parse_value(result) if parse_types else result - - def set(self, key: str, value, all: bool = False): - if all: - nested_update(self.parsed_content, key, value, in_place=True) - elif "." in key: - indexes = split_on_dot(key) - indexes_length = len(indexes) - - ref = self.parsed_content - for index, key in enumerate(indexes): - if index == indexes_length - 1: - ref[key] = value - break - - ref = ref[key] - else: - self.parsed_content[key] = value - - return True - - def delete(self, key, all: bool = False): - if all: - nested_delete(self.parsed_content, key, in_place=True) - elif "." in key: - indexes = split_on_dot(key) - indexes_length = len(indexes) - - ref = self.parsed_content - for index, key in enumerate(indexes): - if index == indexes_length - 1: - del ref[key] - break - - ref = ref[key] - else: - del self.parsed_content[key] - - return True + super().__init__( + file_contents=file_contents, + module=json, + decode_error=json.decoder.JSONDecodeError, + ) def stringify(self) -> str: return json.dumps(self.parsed_content, indent=4) - - def has(self, search_key: str) -> bool: - return get_occurrence_of_key(self.parsed_content, key=search_key) > 0 -- cgit v1.2.3 From 92593a3895c8e045e059d6371027aa99e39a153f Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Mon, 20 Jul 2020 22:53:38 -0500 Subject: Add MissingKeyError --- config_file/__init__.py | 4 +++- config_file/exceptions.py | 15 +++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) (limited to 'config_file') diff --git a/config_file/__init__.py b/config_file/__init__.py index 668c0a9..3825140 100644 --- a/config_file/__init__.py +++ b/config_file/__init__.py @@ -2,10 +2,11 @@ from config_file.config_file import ConfigFile from config_file.exceptions import ( ConfigFileError, MissingDependencyError, + MissingKeyError, ParsingError, UnrecognizedFileError, ) -from config_file.parsers import AbstractParser +from config_file.parsers.abstract_parser import AbstractParser __version__ = "0.10.0" @@ -16,4 +17,5 @@ __all__ = [ "ParsingError", "UnrecognizedFileError", "MissingDependencyError", + "MissingKeyError", ] diff --git a/config_file/exceptions.py b/config_file/exceptions.py index 03ef052..d11e373 100644 --- a/config_file/exceptions.py +++ b/config_file/exceptions.py @@ -2,10 +2,6 @@ class ConfigFileError(Exception): """A generic error from the package.""" -class ParsingError(ConfigFileError): - """Unable to parse the configuration file.""" - - class MissingDependencyError(ConfigFileError): """ PyYaml or toml is not installed, but the parsers @@ -17,3 +13,14 @@ class UnrecognizedFileError(ConfigFileError): """ A missing file extension or a file type not supported. """ + + +class ParsingError(ConfigFileError): + """Unable to parse the configuration file.""" + + +class MissingKeyError(ParsingError): + """ + A key that was specified (to retrieve, delete, etc.) + is not in the file. + """ -- cgit v1.2.3 From 9010575de1cd91c9fb4152835704931255e3f30d Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Mon, 20 Jul 2020 22:54:03 -0500 Subject: Fix type hints and add Default class --- config_file/utils.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'config_file') diff --git a/config_file/utils.py b/config_file/utils.py index 4a83a40..de31c69 100644 --- a/config_file/utils.py +++ b/config_file/utils.py @@ -1,8 +1,8 @@ -from pathlib import Path, PurePath -from typing import Type, Union +from pathlib import Path +from typing import List, Union -def split_on_dot(line: Union[str, Type[PurePath]], only_last_dot=False) -> str: +def split_on_dot(line: Union[str, Path], only_last_dot=False) -> List[str]: """ Split a string on the dot character (.). @@ -11,7 +11,7 @@ def split_on_dot(line: Union[str, Type[PurePath]], only_last_dot=False) -> str: :raises ValueError: if the line does not have a dot. """ - if isinstance(line, PurePath): + if isinstance(line, Path): line = str(line) if "." not in line: @@ -20,12 +20,12 @@ def split_on_dot(line: Union[str, Type[PurePath]], only_last_dot=False) -> str: return line.rsplit(".", 1) if only_last_dot else line.split(".") -def read_file(path: Union[str, Type[PurePath]]): +def read_file(path: Union[str, Path]) -> str: with open(path, "r") as file: return file.read() -def create_config_path(file_path: Type[PurePath], original: bool = False) -> Path: +def create_config_path(file_path: Path, original: bool = False) -> Path: if len(file_path.parts) >= 1 and file_path.parts[0] == "~": return file_path.expanduser() @@ -38,3 +38,11 @@ def create_config_path(file_path: Type[PurePath], original: bool = False) -> Pat raise ValueError(f"The specified config file ({file_path}) is a directory.") return file_path + + +class Default: + def __init__(self, value): + self.value = value + + def __repr__(self): + return "Default Value: {} ({})".format(self.value, type(self.value)) -- cgit v1.2.3 From 7929171083f9d79e4e6b62d7f09189e4de520fe4 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Mon, 20 Jul 2020 22:54:55 -0500 Subject: Remove `all`, start convert to google docstring, add other parsers --- config_file/config_file.py | 168 +++++++++++++++++++++++++++++---------------- 1 file changed, 110 insertions(+), 58 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index 34df561..3ac4b91 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -1,18 +1,23 @@ import inspect -from pathlib import Path, PurePath +from pathlib import Path from shutil import copyfile -from typing import Any, Type, Union +from typing import Any, Optional, Type, Union -from config_file.exceptions import ParsingError -from config_file.parsers import AbstractParser, IniParser, JsonParser -from config_file.utils import create_config_path, read_file, split_on_dot +from config_file.exceptions import ParsingError, UnrecognizedFileError +from config_file.parsers.abstract_parser import AbstractParser +from config_file.parsers.ini_parser import IniParser +from config_file.parsers.json_parser import JsonParser +from config_file.parsers.parse_value import parse_value +from config_file.parsers.toml_parser import TomlParser +from config_file.parsers.yaml_parser import YamlParser +from config_file.utils import Default, create_config_path, read_file, split_on_dot class ConfigFile: def __init__( self, - file_path: Type[Union[str, Type[PurePath]]], - parser: Type[AbstractParser] = None, + file_path: Union[str, Path], + parser: Optional[Type[AbstractParser]] = None, ) -> None: """ Saves the config file path and expands it if needed, reads in @@ -33,23 +38,42 @@ class ConfigFile: self.__parser = self.__determine_parser(file_path, parser) @property - def path(self) -> Type[PurePath]: + def path(self) -> Path: return self.__path def __determine_parser( - self, file_path: Type[PurePath], parser: Type[AbstractParser] = None + self, file_path: Path, parser: Optional[Type[AbstractParser]] = None ) -> Type[AbstractParser]: if parser is not None and not inspect.isabstract(parser): return parser(self.__contents) - file_type = split_on_dot(file_path, only_last_dot=True)[-1] + try: + file_type = split_on_dot(file_path, only_last_dot=True)[-1] + except ValueError: + raise UnrecognizedFileError( + "Tried to determine a parser to use, but the file at " + f"{file_path} does not have an extension." + ) + + return self.__find_parser_by_file_type( + file_type=file_type, file_contents=self.__contents, file_path=self.__path + ) + + @staticmethod + def __find_parser_by_file_type( + file_type: str, file_path: Union[str, Path], file_contents: str + ) -> Type[AbstractParser]: if file_type == "ini": - return IniParser(self.__contents) + return IniParser(file_contents) elif file_type == "json": - return JsonParser(self.__contents) + return JsonParser(file_contents) + elif file_type == "yaml" or file_type == "yml": + return YamlParser(file_contents) + elif file_type == "toml": + return TomlParser(file_contents) else: - raise ValueError( - f"File path contains an unrecognized file type: {file_path}" + raise UnrecognizedFileError( + f"File path at `{file_path}` contains an unrecognized file type." ) def get( @@ -57,8 +81,7 @@ class ConfigFile: key: str, parse_types: bool = False, return_type: Any = None, - default: Any = None, - all: bool = False, + default: Any = Default(None), ) -> Any: """ Retrieve the value of a key. @@ -76,54 +99,83 @@ class ConfigFile: :raises ValueError: If the value is not able to be coerced into return_type """ - if default is None: - key_value = self.__parser.get(key, parse_types=parse_types) - else: - try: - key_value = self.__parser.get(key, parse_types=parse_types) - except ParsingError: - return default + try: + key_value = self.__parser.get(key) + except ParsingError as error: + if default is not type(Default): + key_value = default + else: + raise ParsingError(error) + + if parse_types: + key_value = parse_value(key_value) return return_type(key_value) if return_type else key_value - def set(self, key: str, value: Any) -> bool: - """Sets the value of a key.""" - return self.__parser.set(key, value) + def set(self, key: str, value: Any) -> None: + """Sets the value of a key. - def delete(self, section_key: str) -> bool: - """Deletes a key/value pair or entire sections.""" - return self.__parser.delete(section_key) + """ + self.__parser.set(key, value) + + def delete(self, section_key: str) -> None: + """Deletes a section or key. + + Args: + section_key: + """ + self.__parser.delete(section_key) def stringify(self) -> str: - """Convert the parsed internal representation of the file back into a string.""" + """Retrieves file contents as a string. + + Returns: + The internal representation of the file + that has been read in converted to a string. + """ return self.__parser.stringify() def has(self, section_key: str) -> bool: """ - Check if a section, sub-section, or key exists in the file - using a section.key format. + Check if a section, sub-section, or key exists. + + Some formats, like JSON, do not have sections and + therefore, it would only be checking if a particular + key exists. - Some formats, like JSON, do not have sections and therefore, - it would only be checking if the key exists. + Args: + section_key: + + Returns: + True if """ return self.__parser.has(section_key) def restore_original( - self, original_file_path: Union[str, Type[PurePath]] = None - ) -> bool: - """ - Restores the original the config file by deleting it and copying the original - back in its place. The internal contents of this config file object are then set - to the newly reset config file. - - :param original_file_path: The path to the original config file to reset it to. - If this is not provided and, say, your configuration file is config.json, the - method would look for a config.original.json in the same folder by default. - - :return: True if the reset succeeded - :raises FileNotFoundError: If the original configuration file does not exist - :raises OSError: If self.path is not writable - :raises SameFileError: If self.path and original_file_path are the same file. + self, original_file_path: Union[str, Path, None] = None + ) -> None: + """Restores the original the configuration file. + + The current one is deleted and the original is copied back + in its place. The internal contents are then reset to the + new file. + + Args: + original_file_path: The original file to reset to. + + If this is not provided, it is calculated for you. + e.g. if your configuration file is named config.json, + this will look for aa config.original.json in the same + directory. + + Raises: + FileNotFoundError: If the original configuration file (whether + calculated or passed in) does not exist. + + OSError: If the current configuration path is not writable. + + SameFileError: If current configuration path and the passed in + original_file_path are the same file. """ if original_file_path is None: original_file_path = create_config_path(self.__path, original=True) @@ -135,17 +187,17 @@ class ConfigFile: self.__path.expanduser().unlink() copyfile(original_file_path, self.__path) - self.__contents = read_file(self.__path) - self.__parser.parsed_content = self.__parser.parse(self.__contents) - return True + self.__contents = read_file(self.__path) + self.__parser.reset_internal_contents(self.__contents) - def save(self) -> bool: + def save(self) -> None: """ - Save the configuration changes by writing the file out to the specified path - :return: True if the save succeeded. + Save your configuration changes. + + This writes the file back out, including any changes + you've made, to the specified path when this object + was created. """ - with open(self.__path, "w") as config_file: + with open(str(self.__path), "w") as config_file: config_file.write(self.__parser.stringify()) - - return True -- cgit v1.2.3 From 18306b975e7d53ad58a6039e51741d2f75a7edbd Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Mon, 20 Jul 2020 22:56:14 -0500 Subject: Lint nested_lookup --- config_file/nested_lookup/__init__.py | 21 ++++++++++++++------- config_file/nested_lookup/lookup_api.py | 12 +++--------- config_file/nested_lookup/nested_lookup.py | 10 ++++------ 3 files changed, 21 insertions(+), 22 deletions(-) (limited to 'config_file') diff --git a/config_file/nested_lookup/__init__.py b/config_file/nested_lookup/__init__.py index 768facc..4493bdf 100644 --- a/config_file/nested_lookup/__init__.py +++ b/config_file/nested_lookup/__init__.py @@ -1,12 +1,19 @@ +from .lookup_api import nested_alter, nested_delete, nested_update from .nested_lookup import ( - nested_lookup, get_all_keys, get_occurrence_of_key, get_occurrence_of_value, - get_occurrences_and_values -) -from .lookup_api import ( - nested_update, - nested_delete, - nested_alter + get_occurrences_and_values, + nested_lookup, ) + +__all__ = [ + "get_all_keys", + "get_occurrences_and_values", + "get_occurrence_of_value", + "get_occurrence_of_key", + "nested_update", + "nested_delete", + "nested_alter", + "nested_lookup", +] diff --git a/config_file/nested_lookup/lookup_api.py b/config_file/nested_lookup/lookup_api.py index e103534..886e8bf 100644 --- a/config_file/nested_lookup/lookup_api.py +++ b/config_file/nested_lookup/lookup_api.py @@ -65,9 +65,7 @@ def nested_update(document, key, value, in_place=False, treat_as_element=True): # from the scalar value # check the length of the list and provide it to _nested_update if not treat_as_element and not isinstance(value, list): - raise Exception( - "The value must be a list when treat_as_element is False." - ) + raise Exception("The value must be a list when treat_as_element is False.") elif treat_as_element: value = [value] @@ -91,18 +89,14 @@ def _nested_update(document, key, value): """ if isinstance(document, list): for list_items in document: - _nested_update( - document=list_items, key=key, value=value - ) + _nested_update(document=list_items, key=key, value=value) elif isinstance(document, dict): for dict_key, dict_value in document.items(): if dict_key == key: document[key] = value[0] if len(value) > 1: value.pop(0) - _nested_update( - document=dict_value, key=key, value=value - ) + _nested_update(document=dict_value, key=key, value=value) return document diff --git a/config_file/nested_lookup/nested_lookup.py b/config_file/nested_lookup/nested_lookup.py index 9454a44..6f2e7f2 100644 --- a/config_file/nested_lookup/nested_lookup.py +++ b/config_file/nested_lookup/nested_lookup.py @@ -1,6 +1,5 @@ from collections import defaultdict - values_list = [] @@ -98,15 +97,14 @@ def get_occurrences_and_values(items, value): value_list = [] for item in items: - occurrence_result, values = _get_occurrence_with_values(dictionary=item, item="value", keyword=value) + occurrence_result, values = _get_occurrence_with_values( + dictionary=item, item="value", keyword=value + ) occurrence = occurrence + occurrence_result if occurrence_result: value_list.extend(values) - occurrences[value] = { - 'occurrences': occurrence, - 'values': value_list - } + occurrences[value] = {"occurrences": occurrence, "values": value_list} return occurrences -- cgit v1.2.3 From d8ee572672c6d5429257664c5339b23e0c84c55a Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Wed, 29 Jul 2020 20:54:42 -0500 Subject: Move IniParserCompatibility to new file --- config_file/parsers/ini_parser.py | 44 ++------------------------------ config_file/parsers/ini_parser_compat.py | 42 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 42 deletions(-) create mode 100644 config_file/parsers/ini_parser_compat.py (limited to 'config_file') diff --git a/config_file/parsers/ini_parser.py b/config_file/parsers/ini_parser.py index 2a29ba7..8dc7f47 100644 --- a/config_file/parsers/ini_parser.py +++ b/config_file/parsers/ini_parser.py @@ -1,47 +1,7 @@ import configparser -from io import StringIO from .base_parser import BaseParser - - -class IniParserCompatibility: - """ - Provides compatibility with the json, yaml, - and toml API so we can use the BaseParser - easily, while still being able to leverage - the built-in configparser to parse the file. - - It is used as the `module` parameter - for BaseParser, which requires us to have a - `dumps` and `loads` method. - """ - - @staticmethod - def loads(file_contents: str) -> dict: - parser = configparser.ConfigParser() - parser.read_string(file_contents) - - return IniParserCompatibility.__create_configparser_dict(parser) - - @staticmethod - def dumps(file_contents: dict) -> str: - parser = configparser.ConfigParser() - buffer = StringIO() - - parser.read_dict(file_contents) - parser.write(buffer) - - return buffer.getvalue() - - @staticmethod - def __create_configparser_dict(parser: configparser.ConfigParser) -> dict: - result = {} - items = dict(parser.items()) - - for section in parser.sections(): - result.update({str(section): dict(items[section])}) - - return result +from .ini_parser_compat import IniParserCompatibility class IniParser(BaseParser): @@ -50,4 +10,4 @@ class IniParser(BaseParser): file_contents=file_contents, module=IniParserCompatibility, decode_error=configparser.Error, - ) + ) \ No newline at end of file diff --git a/config_file/parsers/ini_parser_compat.py b/config_file/parsers/ini_parser_compat.py new file mode 100644 index 0000000..9dfea2e --- /dev/null +++ b/config_file/parsers/ini_parser_compat.py @@ -0,0 +1,42 @@ +import configparser +from io import StringIO + + +class IniParserCompatibility: + """ + Provides compatibility with the json, yaml, + and toml API so we can use the BaseParser + easily, while still being able to leverage + the built-in configparser to parse the file. + + It is used as the `module` parameter + for BaseParser, which requires us to have a + `dumps` and `loads` method. + """ + + @staticmethod + def loads(file_contents: str) -> dict: + parser = configparser.ConfigParser() + parser.read_string(file_contents) + + return IniParserCompatibility.__create_configparser_dict(parser) + + @staticmethod + def dumps(file_contents: dict) -> str: + parser = configparser.ConfigParser() + buffer = StringIO() + + parser.read_dict(file_contents) + parser.write(buffer) + + return buffer.getvalue() + + @staticmethod + def __create_configparser_dict(parser: configparser.ConfigParser) -> dict: + result = {} + items = dict(parser.items()) + + for section in parser.sections(): + result.update({str(section): dict(items[section])}) + + return result -- cgit v1.2.3 From 5d78f7e5e8f24476ec21cf2b01c88a62e3acc9ba Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Wed, 29 Jul 2020 21:31:35 -0500 Subject: Revise docstrings, add `wild` to `has` method --- config_file/config_file.py | 96 ++++++++++++++++++++++++++++++---------------- 1 file changed, 63 insertions(+), 33 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index 3ac4b91..9b14072 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -17,25 +17,26 @@ class ConfigFile: def __init__( self, file_path: Union[str, Path], - parser: Optional[Type[AbstractParser]] = None, + parser: Optional[Type[AbstractParser]] = None ) -> None: """ - Saves the config file path and expands it if needed, reads in + Stores the config file path and expands it if needed, reads in the file contents, and determines what parser should be used for the given file. - :param file_path: The path to your configuration file. - :param parser: A custom parser you'd like used for your config file. + Args: + file_path: The path to your configuration file. + parser: A custom parser you'd like used for your config file. - :raises ValueError: If the specified file path does not have an extension - that is supported or it is a directory. + Raises: + ValueError: If the specified file path does not have an extension + that is supported or it is a directory. """ if not isinstance(file_path, Path): file_path = Path(file_path) self.__path = create_config_path(file_path) - self.__contents = read_file(self.__path) - self.__parser = self.__determine_parser(file_path, parser) + self.__parser = self.__determine_parser(self.__path, parser) @property def path(self) -> Path: @@ -44,8 +45,10 @@ class ConfigFile: def __determine_parser( self, file_path: Path, parser: Optional[Type[AbstractParser]] = None ) -> Type[AbstractParser]: + file_contents = read_file(file_path) + if parser is not None and not inspect.isabstract(parser): - return parser(self.__contents) + return parser(file_contents) try: file_type = split_on_dot(file_path, only_last_dot=True)[-1] @@ -56,7 +59,7 @@ class ConfigFile: ) return self.__find_parser_by_file_type( - file_type=file_type, file_contents=self.__contents, file_path=self.__path + file_type=file_type, file_contents=file_contents, file_path=file_path ) @staticmethod @@ -86,18 +89,23 @@ class ConfigFile: """ Retrieve the value of a key. - :param key: The key to retrieve. - :param parse_types: Automatically parse ints, floats, booleans, dicts, and - lists. This recursively parses all types in whatever you're - retrieving, not just a single type. e.g. If you are - retrieving a section, all values in that section with be - parsed. - :param return_type: The type to coerce the return value to. - :param default: The default value to return if the value of the key is empty. + Args: + key: The key to retrieve. + parse_types: Automatically parse ints, floats, booleans, dicts, and + lists. This recursively parses all types in whatever you're + retrieving, not just a single type. + + e.g. If you are retrieving a section, all values in that section + will be parsed. - :return: The value of the key. + return_type: The type to coerce the return value to. + default: The default value to return if the value of the key is empty. - :raises ValueError: If the value is not able to be coerced into return_type + Returns: + The value of the key. + + Raises: + ValueError: If the value is not able to be coerced into return_type. """ try: key_value = self.__parser.get(key) @@ -115,16 +123,33 @@ class ConfigFile: def set(self, key: str, value: Any) -> None: """Sets the value of a key. + If the given key does not exist, it will be automatically + created for you. That includes if there are multiple keys + in a row that do not exist. + e.g. set('exists.does_not.does_not.also_does_not', 5) + + The behavior of how this is done, however, depends on the + file used. For example, with INI, subsections are not supported. + So it would create a key in the section `exists` called `does_not` + and set it to the value {'does_not': {'also_does_not': 5}}. + + Args: + key: The section, sub-section, or key to delete. + value: The value to set the key to. """ self.__parser.set(key, value) - def delete(self, section_key: str) -> None: + def delete(self, key: str) -> None: """Deletes a section or key. Args: - section_key: + key: The section, sub-section, or key to delete. + + Raises: + MissingKeyError: If a key is attempted to be deleted that + does not exist. """ - self.__parser.delete(section_key) + self.__parser.delete(key) def stringify(self) -> str: """Retrieves file contents as a string. @@ -135,7 +160,7 @@ class ConfigFile: """ return self.__parser.stringify() - def has(self, section_key: str) -> bool: + def has(self, key: str, wild: bool = False) -> bool: """ Check if a section, sub-section, or key exists. @@ -144,12 +169,19 @@ class ConfigFile: key exists. Args: - section_key: + key: The section, sub-section, or key to find. + wild: Whether or not to search everywhere. + + Without `wild`, a single word `key` without a `.` + will look at the outer most heirarchy of the file for it. + + With `wild`, that single word `key` will be searched + for throughout the entire file. Returns: - True if + True if the key exists. False otherwise. """ - return self.__parser.has(section_key) + return self.__parser.has(key, wild=wild) def restore_original( self, original_file_path: Union[str, Path, None] = None @@ -187,17 +219,15 @@ class ConfigFile: self.__path.expanduser().unlink() copyfile(original_file_path, self.__path) - - self.__contents = read_file(self.__path) - self.__parser.reset_internal_contents(self.__contents) + self.__parser.reset_internal_contents(read_file(self.__path)) def save(self) -> None: """ Save your configuration changes. This writes the file back out, including any changes - you've made, to the specified path when this object - was created. + you've made, to the specified path given from this + object's constructor. """ with open(str(self.__path), "w") as config_file: - config_file.write(self.__parser.stringify()) + config_file.write(self.stringify()) -- cgit v1.2.3 From 5b9cbecb01b3ac11f553a62d3ee6135a789943a7 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Wed, 29 Jul 2020 21:32:00 -0500 Subject: Remove duplication from `can_be_parsed_as_float` --- config_file/parsers/parse_value.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'config_file') diff --git a/config_file/parsers/parse_value.py b/config_file/parsers/parse_value.py index 26bdf2e..5f6dfe3 100644 --- a/config_file/parsers/parse_value.py +++ b/config_file/parsers/parse_value.py @@ -75,7 +75,7 @@ def can_be_parsed_as_float(value) -> bool: return False if value.startswith("-"): - return bool(re.match(FLOAT_REGEX, value[1:])) + value = value[1:] return bool(re.match(FLOAT_REGEX, value)) -- cgit v1.2.3 From 47a58715546d780a5df8d362ab5eeedd3813ecdb Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Wed, 29 Jul 2020 21:32:57 -0500 Subject: Use ruamel.yaml for yaml parser --- config_file/parsers/yaml_parser.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/yaml_parser.py b/config_file/parsers/yaml_parser.py index 77b59fe..1937e17 100644 --- a/config_file/parsers/yaml_parser.py +++ b/config_file/parsers/yaml_parser.py @@ -1,12 +1,7 @@ try: - import yaml + from ruamel.yaml import YAML, YAMLError, round_trip_dump, round_trip_load YAML_AVAILABLE = True - - # Conform yaml to json and - # toml package syntax - yaml.loads = yaml.load - yaml.dumps = yaml.dump except ImportError: YAML_AVAILABLE = False @@ -23,7 +18,11 @@ class YamlParser(BaseParser): "file was attempted to be used. Install the `toml` " "extra first with `pip install config-file[toml]`." ) + + yaml = YAML(typ='safe') + yaml.loads = round_trip_load + yaml.dumps = round_trip_dump super().__init__( - file_contents=file_contents, module=yaml, decode_error=yaml.YAMLError + file_contents=file_contents, module=yaml, decode_error=YAMLError ) -- cgit v1.2.3 From 03811338b3bc9d07d491454ba5646ba2dd912463 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Wed, 29 Jul 2020 21:33:54 -0500 Subject: Fix `get`, `set`, `delete` and `has` in BaseParser --- config_file/parsers/base_parser.py | 39 ++++++++++++++++++++++++++------------ config_file/parsers/toml_parser.py | 1 - 2 files changed, 27 insertions(+), 13 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/base_parser.py b/config_file/parsers/base_parser.py index 09fbbbf..546450c 100644 --- a/config_file/parsers/base_parser.py +++ b/config_file/parsers/base_parser.py @@ -1,4 +1,5 @@ from typing import Any +from types import ModuleType from config_file.exceptions import MissingKeyError, ParsingError from config_file.nested_lookup import get_occurrence_of_key @@ -8,7 +9,9 @@ from .abstract_parser import AbstractParser class BaseParser(AbstractParser): - def __init__(self, file_contents: str, module, decode_error): + def __init__( + self, file_contents: str, module: ModuleType, decode_error: Exception + ): super().__init__(file_contents) self.__module = module self.__decode_error = decode_error @@ -39,9 +42,9 @@ class BaseParser(AbstractParser): for key in split_keys: key_causing_error = key result = result[key] - except TypeError: + except (KeyError, TypeError): raise ParsingError( - f"Cannot get {search_key} because " + f"Cannot `get` {search_key} because " f"{key_causing_error} is not subscriptable." ) @@ -59,9 +62,13 @@ class BaseParser(AbstractParser): content_reference[key] = value return - content_reference = content_reference[key] + try: + content_reference = content_reference[key] + except KeyError: + content_reference[key] = {} + content_reference = content_reference[key] - def delete(self, key) -> None: + def delete(self, key: str) -> None: try: if "." not in key: del self.parsed_content[key] @@ -69,18 +76,26 @@ class BaseParser(AbstractParser): indexes = split_on_dot(key) indexes_length = len(indexes) - ref = self.parsed_content + content_reference = self.parsed_content for index, key in enumerate(indexes): if index == indexes_length - 1: - del ref[key] + del content_reference[key] break - ref = ref[key] - except KeyError: - raise MissingKeyError(f"The specified key '{key}' was not found.") + content_reference = content_reference[key] + except (KeyError, TypeError): + raise MissingKeyError(f"The specified key '{key}' to delete was not found.") def stringify(self) -> str: return self.__module.dumps(self.parsed_content) - def has(self, search_key: str) -> bool: - return get_occurrence_of_key(self.parsed_content, key=search_key) > 0 + def has(self, search_key: str, wild: bool = False) -> bool: + if wild: + return get_occurrence_of_key(self.parsed_content, key=search_key) > 0 + + try: + self.get(search_key) + return True + except (KeyError, ParsingError): + return False + diff --git a/config_file/parsers/toml_parser.py b/config_file/parsers/toml_parser.py index d369003..58e20b9 100644 --- a/config_file/parsers/toml_parser.py +++ b/config_file/parsers/toml_parser.py @@ -9,7 +9,6 @@ from config_file.exceptions import MissingDependencyError from .base_parser import BaseParser - class TomlParser(BaseParser): def __init__(self, file_contents: str): if not TOML_AVAILABLE: -- cgit v1.2.3 From 3c1ebfaf0db4a29cc8ee9ba742c6d8925dc79cff Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Wed, 29 Jul 2020 23:53:15 -0500 Subject: Allow string lists and dicts to be auto parsed --- config_file/parsers/parse_value.py | 61 ++++++++++++++++++++++++++++++++------ config_file/utils.py | 8 +++-- 2 files changed, 57 insertions(+), 12 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/parse_value.py b/config_file/parsers/parse_value.py index 5f6dfe3..aa8d1fe 100644 --- a/config_file/parsers/parse_value.py +++ b/config_file/parsers/parse_value.py @@ -19,20 +19,12 @@ Example: can_be_parsed_as_int("-5") -> True parse_value("-5") -> -5 """ +import ast import re from distutils.util import strtobool def parse_value(value): - if can_be_parsed_as_int(value): - return int(value) - - if can_be_parsed_as_float(value): - return float(value) - - if can_be_parsed_as_bool(value): - return value if type(value) is bool else bool(strtobool(value)) - if type(value) is dict: parsed = {} @@ -49,6 +41,21 @@ def parse_value(value): return parsed + if can_be_parsed_as_int(value): + return int(value) + + if can_be_parsed_as_float(value): + return float(value) + + if can_be_parsed_as_bool(value): + return value if type(value) is bool else bool(strtobool(value)) + + if can_be_parsed_as_dict(value): + return parse_value(ast.literal_eval(value)) + + if can_be_parsed_as_list(value): + return parse_value(ast.literal_eval(value)) + return value @@ -88,3 +95,39 @@ def can_be_parsed_as_bool(value) -> bool: return False return value.lower() == "true" or value.lower() == "false" + + +def can_be_parsed_as_dict(value) -> bool: + if type(value) is dict: + return True + + if type(value) is not str: + return False + + value = value.strip() + if value[0] == "{" and value[-1] == "}": + try: + ast.literal_eval(value) + return True + except SyntaxError: + return False + + return False + + +def can_be_parsed_as_list(value) -> bool: + if type(value) is dict: + return True + + if type(value) is not str: + return False + + value = value.strip() + if value[0] == "[" and value[-1] == "]": + try: + ast.literal_eval(value) + return True + except SyntaxError: + return False + + return False diff --git a/config_file/utils.py b/config_file/utils.py index de31c69..a75968c 100644 --- a/config_file/utils.py +++ b/config_file/utils.py @@ -6,10 +6,12 @@ def split_on_dot(line: Union[str, Path], only_last_dot=False) -> List[str]: """ Split a string on the dot character (.). - :param line: The line ot split on. - :param only_last_dot: Only split on the last occurrence of the dot. + Args: + line: The line ot split on. + only_last_dot: Only split on the last occurrence of the dot. - :raises ValueError: if the line does not have a dot. + Raises: + ValueError: if the line does not have a dot. """ if isinstance(line, Path): line = str(line) -- cgit v1.2.3 From e40a3a23fb6649758f40e9b67e3f3100ec23f116 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Thu, 30 Jul 2020 12:57:56 -0500 Subject: Fix missing dependency error messages for yaml and toml --- config_file/parsers/toml_parser.py | 3 ++- config_file/parsers/yaml_parser.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/toml_parser.py b/config_file/parsers/toml_parser.py index 58e20b9..1c7f8a7 100644 --- a/config_file/parsers/toml_parser.py +++ b/config_file/parsers/toml_parser.py @@ -9,11 +9,12 @@ from config_file.exceptions import MissingDependencyError from .base_parser import BaseParser + class TomlParser(BaseParser): def __init__(self, file_contents: str): if not TOML_AVAILABLE: raise MissingDependencyError( - "It doesn't appear `toml` is installed, but a toml " + "It doesn't appear `tomlkit` is installed, but a toml " "file was attempted to be used. Install the `toml` " "extra first with `pip install config-file[toml]`." ) diff --git a/config_file/parsers/yaml_parser.py b/config_file/parsers/yaml_parser.py index 1937e17..babe873 100644 --- a/config_file/parsers/yaml_parser.py +++ b/config_file/parsers/yaml_parser.py @@ -14,12 +14,12 @@ class YamlParser(BaseParser): def __init__(self, file_contents: str): if not YAML_AVAILABLE: raise MissingDependencyError( - "It doesn't appear `PyYaml` is installed, but a yaml " - "file was attempted to be used. Install the `toml` " - "extra first with `pip install config-file[toml]`." + "It doesn't appear `ruamel.yaml` is installed, but a yaml " + "file was attempted to be used. Install the `yaml` " + "extra first with `pip install config-file[yaml]`." ) - - yaml = YAML(typ='safe') + + yaml = YAML(typ="safe") yaml.loads = round_trip_load yaml.dumps = round_trip_dump -- cgit v1.2.3 From 3700e247d5b72f21c20c554c7f2a72d6c63b8eea Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Thu, 30 Jul 2020 20:34:27 -0500 Subject: Fix wording of docstring in missing dependency error --- config_file/exceptions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'config_file') diff --git a/config_file/exceptions.py b/config_file/exceptions.py index d11e373..d1b466b 100644 --- a/config_file/exceptions.py +++ b/config_file/exceptions.py @@ -4,8 +4,8 @@ class ConfigFileError(Exception): class MissingDependencyError(ConfigFileError): """ - PyYaml or toml is not installed, but the parsers - for them were attempted to be used. + yaml or toml dependencies were not installed, + but the parsers for them were attempted to be used. """ -- cgit v1.2.3 From 5ac52d231e56f60a22ba3f755415c4aa43a14d83 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 1 Aug 2020 11:20:39 -0500 Subject: Add tests to parse_value and fix bugs in it --- config_file/parsers/parse_value.py | 78 ++++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 25 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/parse_value.py b/config_file/parsers/parse_value.py index aa8d1fe..28f1ccb 100644 --- a/config_file/parsers/parse_value.py +++ b/config_file/parsers/parse_value.py @@ -22,10 +22,22 @@ parse_value("-5") -> -5 import ast import re from distutils.util import strtobool +from typing import Any -def parse_value(value): - if type(value) is dict: +def parse_value(value: Any) -> Any: + """Parse a value into native type. + + Args: + value: The value to parse. + + Returns: + The parsed value, if it could be parsed + as a dictionary, list, int, float, or bool. + Otherwise, it simply returns the original + passed in value. + """ + if isinstance(value, dict): parsed = {} for item in value: @@ -33,7 +45,7 @@ def parse_value(value): return parsed - if type(value) is list: + if isinstance(value, list): parsed = [] for item in value: @@ -42,92 +54,108 @@ def parse_value(value): return parsed if can_be_parsed_as_int(value): - return int(value) + return int(value.strip()) if isinstance(value, str) else int(value) if can_be_parsed_as_float(value): - return float(value) + return float(value.strip()) if isinstance(value, str) else float(value) if can_be_parsed_as_bool(value): - return value if type(value) is bool else bool(strtobool(value)) + return bool(value) if isinstance(value, bool) else bool(strtobool(value)) if can_be_parsed_as_dict(value): - return parse_value(ast.literal_eval(value)) + return parse_value(ast.literal_eval(value.strip())) if can_be_parsed_as_list(value): - return parse_value(ast.literal_eval(value)) + return parse_value(ast.literal_eval(value.strip())) return value -def can_be_parsed_as_int(value) -> bool: - if type(value) is int: +def can_be_parsed_as_int(value: Any) -> bool: + """Check whether a value can be parsed as a integer. + + Args: + value: The value to check if it can be parsed as an integer. + + Returns: + True if we can parse the value as a integer. False otherwise. + """ + if isinstance(value, int) and not type(value) is bool: return True - if type(value) is not str: + if not isinstance(value, str): return False + value = value.strip() + if value.startswith("-"): return value[1:].isdigit() return value.isdigit() -def can_be_parsed_as_float(value) -> bool: +def can_be_parsed_as_float(value: Any) -> bool: FLOAT_REGEX = r"^\d*\.\d+$" - if type(value) is float: + if isinstance(value, float): return True - if type(value) is not str: + if not isinstance(value, str): return False + value = value.strip() + if value.startswith("-"): value = value[1:] return bool(re.match(FLOAT_REGEX, value)) -def can_be_parsed_as_bool(value) -> bool: - if type(value) is bool: +def can_be_parsed_as_bool(value: Any) -> bool: + if isinstance(value, bool): return True - if type(value) is not str: + if not isinstance(value, str): return False - return value.lower() == "true" or value.lower() == "false" + value = value.lower().strip() + + return value == "true" or value == "false" -def can_be_parsed_as_dict(value) -> bool: - if type(value) is dict: +def can_be_parsed_as_dict(value: Any) -> bool: + if isinstance(value, dict): return True - if type(value) is not str: + if not isinstance(value, str): return False value = value.strip() + if value[0] == "{" and value[-1] == "}": try: ast.literal_eval(value) return True - except SyntaxError: + except Exception: return False return False def can_be_parsed_as_list(value) -> bool: - if type(value) is dict: + if isinstance(value, list): return True - if type(value) is not str: + if not isinstance(value, str): return False value = value.strip() + if value[0] == "[" and value[-1] == "]": try: ast.literal_eval(value) return True - except SyntaxError: + except Exception: return False return False -- cgit v1.2.3 From b1c23487c33fc2ad5503b6340b17980a326918c8 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 1 Aug 2020 11:53:56 -0500 Subject: Lint files --- config_file/config_file.py | 32 +++++++++++++++----------------- config_file/parsers/base_parser.py | 9 +++------ config_file/parsers/ini_parser.py | 2 +- 3 files changed, 19 insertions(+), 24 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index 9b14072..adcfaf1 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -15,9 +15,7 @@ from config_file.utils import Default, create_config_path, read_file, split_on_d class ConfigFile: def __init__( - self, - file_path: Union[str, Path], - parser: Optional[Type[AbstractParser]] = None + self, file_path: Union[str, Path], parser: Optional[Type[AbstractParser]] = None ) -> None: """ Stores the config file path and expands it if needed, reads in @@ -93,10 +91,10 @@ class ConfigFile: key: The key to retrieve. parse_types: Automatically parse ints, floats, booleans, dicts, and lists. This recursively parses all types in whatever you're - retrieving, not just a single type. - - e.g. If you are retrieving a section, all values in that section - will be parsed. + retrieving, not just a single type. + + e.g. If you are retrieving a section, all values in that + section will be parsed. return_type: The type to coerce the return value to. default: The default value to return if the value of the key is empty. @@ -123,14 +121,14 @@ class ConfigFile: def set(self, key: str, value: Any) -> None: """Sets the value of a key. - If the given key does not exist, it will be automatically - created for you. That includes if there are multiple keys + If the given key does not exist, it will be automatically + created for you. That includes if there are multiple keys in a row that do not exist. e.g. set('exists.does_not.does_not.also_does_not', 5) - The behavior of how this is done, however, depends on the - file used. For example, with INI, subsections are not supported. - So it would create a key in the section `exists` called `does_not` + The behavior of how this is done, however, depends on the + file used. For example, with INI, subsections are not supported. + So it would create a key in the section `exists` called `does_not` and set it to the value {'does_not': {'also_does_not': 5}}. Args: @@ -144,7 +142,7 @@ class ConfigFile: Args: key: The section, sub-section, or key to delete. - + Raises: MissingKeyError: If a key is attempted to be deleted that does not exist. @@ -170,9 +168,9 @@ class ConfigFile: Args: key: The section, sub-section, or key to find. - wild: Whether or not to search everywhere. - - Without `wild`, a single word `key` without a `.` + wild: Whether or not to search everywhere. + + Without `wild`, a single word `key` without a `.` will look at the outer most heirarchy of the file for it. With `wild`, that single word `key` will be searched @@ -226,7 +224,7 @@ class ConfigFile: Save your configuration changes. This writes the file back out, including any changes - you've made, to the specified path given from this + you've made, to the specified path given from this object's constructor. """ with open(str(self.__path), "w") as config_file: diff --git a/config_file/parsers/base_parser.py b/config_file/parsers/base_parser.py index 546450c..c2a7d71 100644 --- a/config_file/parsers/base_parser.py +++ b/config_file/parsers/base_parser.py @@ -1,5 +1,5 @@ -from typing import Any from types import ModuleType +from typing import Any from config_file.exceptions import MissingKeyError, ParsingError from config_file.nested_lookup import get_occurrence_of_key @@ -9,9 +9,7 @@ from .abstract_parser import AbstractParser class BaseParser(AbstractParser): - def __init__( - self, file_contents: str, module: ModuleType, decode_error: Exception - ): + def __init__(self, file_contents: str, module: ModuleType, decode_error: Exception): super().__init__(file_contents) self.__module = module self.__decode_error = decode_error @@ -92,10 +90,9 @@ class BaseParser(AbstractParser): def has(self, search_key: str, wild: bool = False) -> bool: if wild: return get_occurrence_of_key(self.parsed_content, key=search_key) > 0 - + try: self.get(search_key) return True except (KeyError, ParsingError): return False - diff --git a/config_file/parsers/ini_parser.py b/config_file/parsers/ini_parser.py index 8dc7f47..72974f1 100644 --- a/config_file/parsers/ini_parser.py +++ b/config_file/parsers/ini_parser.py @@ -10,4 +10,4 @@ class IniParser(BaseParser): file_contents=file_contents, module=IniParserCompatibility, decode_error=configparser.Error, - ) \ No newline at end of file + ) -- cgit v1.2.3 From c7922b08bf010375416138b2ca85fe88fd8edcef Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 1 Aug 2020 11:54:22 -0500 Subject: Expand on docstrings for parse_value file --- config_file/parsers/parse_value.py | 95 ++++++++++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 19 deletions(-) (limited to 'config_file') diff --git a/config_file/parsers/parse_value.py b/config_file/parsers/parse_value.py index 28f1ccb..12032d6 100644 --- a/config_file/parsers/parse_value.py +++ b/config_file/parsers/parse_value.py @@ -1,7 +1,13 @@ -""" -This module allows you to parse a string into its native type. +"""This module allows you to parse a string into its native type.""" +import ast +import re +from distutils.util import strtobool +from typing import Any + + +def parse_value(value: Any) -> Any: + """Parse a value into native type. -Example: parse_value('5') -> 5 (int) parse_value('-5') -> -5 (int) @@ -13,20 +19,7 @@ Example: parse_value('true') -> True (bool) parse_value('FALSE') -> False (bool) - -"5".isdigit() -> True -"-5".isdigit() -> False -can_be_parsed_as_int("-5") -> True -parse_value("-5") -> -5 -""" -import ast -import re -from distutils.util import strtobool -from typing import Any - - -def parse_value(value: Any) -> Any: - """Parse a value into native type. + parse_value(" [ [ [ [] ]] ] ") -> [[[[]]]] (list) Args: value: The value to parse. @@ -74,13 +67,18 @@ def parse_value(value: Any) -> Any: def can_be_parsed_as_int(value: Any) -> bool: """Check whether a value can be parsed as a integer. + can_be_parsed_as_int(5) -> True + can_be_parsed_as_int("5") -> True + can_be_parsed_as_int("-5") -> True + can_be_parsed_as_int(True) -> False + Args: value: The value to check if it can be parsed as an integer. Returns: True if we can parse the value as a integer. False otherwise. """ - if isinstance(value, int) and not type(value) is bool: + if isinstance(value, int) and not isinstance(value, bool): return True if not isinstance(value, str): @@ -95,6 +93,20 @@ def can_be_parsed_as_int(value: Any) -> bool: def can_be_parsed_as_float(value: Any) -> bool: + """Checks whether a value can be parsed as a float. + + can_be_parsed_as_float(45) -> False + can_be_parsed_as_float(1.1) -> True + can_be_parsed_as_float("1.1") -> True + can_be_parsed_as_float("-1.1") -> True + can_be_parsed_as_float(".1") -> True + + Args: + value: The value to be parsed. + + Returns: + True if the value can be parsed as a float. False otherwise. + """ FLOAT_REGEX = r"^\d*\.\d+$" if isinstance(value, float): @@ -112,6 +124,20 @@ def can_be_parsed_as_float(value: Any) -> bool: def can_be_parsed_as_bool(value: Any) -> bool: + """Checks whether a value can be parsed as a boolean. + + can_be_parsed_as_bool(True) -> True + can_be_parsed_as_bool("true") -> True + can_be_parsed_as_bool("false") -> True + can_be_parsed_as_bool("TRUE") -> True + can_be_parsed_as_bool(0) -> False + + Args: + value: The value to be parsed. + + Returns: + True if the value can be parsed as a boolean. False otherwise. + """ if isinstance(value, bool): return True @@ -124,6 +150,21 @@ def can_be_parsed_as_bool(value: Any) -> bool: def can_be_parsed_as_dict(value: Any) -> bool: + """Checks whether a value can be parsed as a dictionary. + + can_be_parsed_as_dict({}) -> True + can_be_parsed_as_dict("{}") -> True + can_be_parsed_as_dict(" { } ") -> True + can_be_parsed_as_dict("{ 'hello': 5 }") -> True + can_be_parsed_as_dict([]]) -> False + can_be_parsed_as_dict(" { invalid } ") -> False + + Args: + value: The value to be parsed. + + Returns: + True if the value can be parsed as a dictionary. False otherwise. + """ if isinstance(value, dict): return True @@ -142,7 +183,23 @@ def can_be_parsed_as_dict(value: Any) -> bool: return False -def can_be_parsed_as_list(value) -> bool: +def can_be_parsed_as_list(value: Any) -> bool: + """Checks whether a value can be parsed as a list. + + can_be_parsed_as_list([]) -> True + can_be_parsed_as_list([1, 2]) -> True + can_be_parsed_as_list("[]") -> True + can_be_parsed_as_list(" [] ") -> True + can_be_parsed_as_list("[1, 2]") -> True + can_be_parsed_as_list({}) -> False + can_be_parsed_as_list(" [ invalid ] ") -> False + + Args: + value: The value to be parsed. + + Returns: + True if the value can be parsed as a list. False otherwise. + """ if isinstance(value, list): return True -- cgit v1.2.3 From 3fa2561f16f684bb83aa9b3dcacb44171b04ddc5 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 1 Aug 2020 11:56:44 -0500 Subject: Raise FileNotFoundError if the config file does not exist --- config_file/utils.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'config_file') diff --git a/config_file/utils.py b/config_file/utils.py index a75968c..4d8bb36 100644 --- a/config_file/utils.py +++ b/config_file/utils.py @@ -39,6 +39,11 @@ def create_config_path(file_path: Path, original: bool = False) -> Path: if file_path.is_dir(): raise ValueError(f"The specified config file ({file_path}) is a directory.") + if not file_path.exists(): + raise FileNotFoundError( + f"The specified config file ({file_path}) does not exist." + ) + return file_path -- cgit v1.2.3 From 032196104a59761bd75b5d594440d1e8ef16965a Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 1 Aug 2020 17:58:32 -0500 Subject: Add ConfigFilePath to refactor out file specific code --- config_file/config_file.py | 74 +++++++------------------ config_file/config_file_path.py | 117 ++++++++++++++++++++++++++++++++++++++++ config_file/default.py | 14 +++++ config_file/utils.py | 33 ------------ 4 files changed, 151 insertions(+), 87 deletions(-) create mode 100644 config_file/config_file_path.py create mode 100644 config_file/default.py (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index adcfaf1..3e8a806 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -3,14 +3,11 @@ from pathlib import Path from shutil import copyfile from typing import Any, Optional, Type, Union -from config_file.exceptions import ParsingError, UnrecognizedFileError +from config_file.config_file_path import ConfigFilePath +from config_file.default import Default +from config_file.exceptions import ParsingError from config_file.parsers.abstract_parser import AbstractParser -from config_file.parsers.ini_parser import IniParser -from config_file.parsers.json_parser import JsonParser from config_file.parsers.parse_value import parse_value -from config_file.parsers.toml_parser import TomlParser -from config_file.parsers.yaml_parser import YamlParser -from config_file.utils import Default, create_config_path, read_file, split_on_dot class ConfigFile: @@ -29,53 +26,22 @@ class ConfigFile: Raises: ValueError: If the specified file path does not have an extension that is supported or it is a directory. + FileNotFoundError: If the specified file path does not exist. """ - if not isinstance(file_path, Path): - file_path = Path(file_path) - - self.__path = create_config_path(file_path) - self.__parser = self.__determine_parser(self.__path, parser) - - @property - def path(self) -> Path: - return self.__path - - def __determine_parser( - self, file_path: Path, parser: Optional[Type[AbstractParser]] = None - ) -> Type[AbstractParser]: - file_contents = read_file(file_path) + self.__path = ConfigFilePath(file_path).validate() if parser is not None and not inspect.isabstract(parser): - return parser(file_contents) + self.__parser = parser(self.__path.contents) + else: + self.__parser = self.__path.parser - try: - file_type = split_on_dot(file_path, only_last_dot=True)[-1] - except ValueError: - raise UnrecognizedFileError( - "Tried to determine a parser to use, but the file at " - f"{file_path} does not have an extension." - ) + @property + def file_path(self) -> Path: + return Path(self.__path) - return self.__find_parser_by_file_type( - file_type=file_type, file_contents=file_contents, file_path=file_path - ) - - @staticmethod - def __find_parser_by_file_type( - file_type: str, file_path: Union[str, Path], file_contents: str - ) -> Type[AbstractParser]: - if file_type == "ini": - return IniParser(file_contents) - elif file_type == "json": - return JsonParser(file_contents) - elif file_type == "yaml" or file_type == "yml": - return YamlParser(file_contents) - elif file_type == "toml": - return TomlParser(file_contents) - else: - raise UnrecognizedFileError( - f"File path at `{file_path}` contains an unrecognized file type." - ) + @property + def original_file_path(self) -> Path: + return Path(self.__path.original_path) def get( self, @@ -207,17 +173,17 @@ class ConfigFile: SameFileError: If current configuration path and the passed in original_file_path are the same file. """ - if original_file_path is None: - original_file_path = create_config_path(self.__path, original=True) - - if not Path(original_file_path).exists(): + if original_file_path and not ConfigFilePath(original_file_path).exists(): raise FileNotFoundError( f"The {original_file_path} file to restore to does not exist." ) + else: + original_file_path = self.__path.original_path + original_file_path = original_file_path.validate() - self.__path.expanduser().unlink() + self.__path.unlink() copyfile(original_file_path, self.__path) - self.__parser.reset_internal_contents(read_file(self.__path)) + self.__parser.reset_internal_contents(self.__path.contents) def save(self) -> None: """ diff --git a/config_file/config_file_path.py b/config_file/config_file_path.py new file mode 100644 index 0000000..a40cc40 --- /dev/null +++ b/config_file/config_file_path.py @@ -0,0 +1,117 @@ +import os +from pathlib import Path as _Path_ +from pathlib import _posix_flavour, _windows_flavour +from typing import Type + +from .exceptions import UnrecognizedFileError +from .parsers.abstract_parser import AbstractParser +from .parsers.ini_parser import IniParser +from .parsers.json_parser import JsonParser +from .parsers.toml_parser import TomlParser +from .parsers.yaml_parser import YamlParser +from .utils import split_on_dot + + +class ConfigFilePath(_Path_): + """A ConfigFilePath object, which subclasses Path. + + It provides us easy ways to grab the original config file + path, file extension, validate the path, or determine the parser + that should be use for the given file type. + + See https://codereview.stackexchange.com/questions/162426/subclassing-pathlib-path + for most details on subclassing the pathlib.Path object. + """ + + _flavour = _windows_flavour if os.name == "nt" else _posix_flavour + + @property + def original_path(self) -> "ConfigFilePath": + """Computes the original configuration file path. + + e.g. ConfigFilePath("./config.ini").original_path -> "./config.original.ini" + + Returns: + The path to the original configuration file, which + may or may not exist. + """ + file_parts = split_on_dot(self, only_last_dot=True) + file_parts.insert(-1, "original") + return ConfigFilePath(".".join(file_parts)) + + @property + def extension(self) -> str: + """Retrieves the extension of the file + path we passed in, without the '.'. + + e.g. ConfigFilePath("./pyproject.toml").extension -> "toml" + + Returns: + The extension of the current file path. + """ + _, extension = os.path.splitext(str(self)) + return extension[1:] + + @property + def parser(self) -> Type[AbstractParser]: + """Determine what parser should be used for this file. + + Raises: + UnrecognizedFileError: If the extension of the file + is not recognized. + + Returns: + The instantiated parser that should be used for this file. + """ + if self.extension == "": + raise UnrecognizedFileError( + "Tried to determine a parser to use, but the file at " + f"{self} does not have an extension." + ) + + if self.extension == "ini": + return IniParser(self.contents) + elif self.extension == "json": + return JsonParser(self.contents) + elif self.extension == "yaml" or self.extension == "yml": + return YamlParser(self.contents) + elif self.extension == "toml": + return TomlParser(self.contents) + else: + raise UnrecognizedFileError( + f"File path at `{self}` contains an unrecognized file type." + ) + + @property + def contents(self) -> str: + """Retrieve the contents of the file. + + Returns: + The contents of the file at this path. + """ + with open(self, "r") as file: + return file.read() + + def validate(self) -> "ConfigFilePath": + """Validates that the current path exists, is a directory, + and expands any home tidles. + + Raises: + FileNotFoundError: If the path does not exist. + ValueError: If the path leads to a directory. + + Returns: + This ConfigFilePath object + """ + if len(self.parts) >= 1 and self.parts[0] == "~": + self = ConfigFilePath(self.expanduser()) + + if not self.exists(): + raise FileNotFoundError( + f"The specified config file ({self}) does not exist." + ) + + if self.is_dir(): + raise ValueError(f"The specified config file ({self}) is a directory") + + return self diff --git a/config_file/default.py b/config_file/default.py new file mode 100644 index 0000000..529021c --- /dev/null +++ b/config_file/default.py @@ -0,0 +1,14 @@ +class Default: + """ + Default is used for the `default` value in ConfigFile's `get`. + + Previously, the default value for `get` was None. However, then + the user cannot have `get` return a default value of None. So this + class is used instead to get around that limitation. + """ + + def __init__(self, value): + self.value = value + + def __repr__(self): + return "Default Value: {} ({})".format(self.value, type(self.value)) diff --git a/config_file/utils.py b/config_file/utils.py index 4d8bb36..c6c3d24 100644 --- a/config_file/utils.py +++ b/config_file/utils.py @@ -20,36 +20,3 @@ def split_on_dot(line: Union[str, Path], only_last_dot=False) -> List[str]: raise ValueError(f"The given string does not have a dot to split on: {line}") return line.rsplit(".", 1) if only_last_dot else line.split(".") - - -def read_file(path: Union[str, Path]) -> str: - with open(path, "r") as file: - return file.read() - - -def create_config_path(file_path: Path, original: bool = False) -> Path: - if len(file_path.parts) >= 1 and file_path.parts[0] == "~": - return file_path.expanduser() - - if original: - file_parts = split_on_dot(file_path, only_last_dot=True) - file_parts.insert(-1, "original") - file_path = Path(".".join(file_parts)) - - if file_path.is_dir(): - raise ValueError(f"The specified config file ({file_path}) is a directory.") - - if not file_path.exists(): - raise FileNotFoundError( - f"The specified config file ({file_path}) does not exist." - ) - - return file_path - - -class Default: - def __init__(self, value): - self.value = value - - def __repr__(self): - return "Default Value: {} ({})".format(self.value, type(self.value)) -- cgit v1.2.3 From 59eeb1321fbad41dce7c971778e37ac1943fda3b Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Wed, 5 Aug 2020 20:39:28 -0500 Subject: Redo package to have a flat package structure By having a flat package structure like this, it'll avoid exposing packages and things we don't want to be exposed. --- config_file/__init__.py | 16 +- config_file/abstract_parser.py | 117 +++++++++ config_file/base_parser.py | 139 ++++++++++ config_file/config_file.py | 6 +- config_file/config_file_path.py | 21 +- config_file/default.py | 14 - config_file/exceptions.py | 2 +- config_file/ini_parser.py | 13 + config_file/ini_parser_compat.py | 42 +++ config_file/json_parser.py | 15 ++ config_file/nested_lookup.py | 402 +++++++++++++++++++++++++++++ config_file/nested_lookup/__init__.py | 19 -- config_file/nested_lookup/lookup_api.py | 214 --------------- config_file/nested_lookup/nested_lookup.py | 178 ------------- config_file/parse_value.py | 218 ++++++++++++++++ config_file/parsers/__init__.py | 0 config_file/parsers/abstract_parser.py | 117 --------- config_file/parsers/base_parser.py | 98 ------- config_file/parsers/ini_parser.py | 13 - config_file/parsers/ini_parser_compat.py | 42 --- config_file/parsers/json_parser.py | 15 -- config_file/parsers/parse_value.py | 218 ---------------- config_file/parsers/toml_parser.py | 28 -- config_file/parsers/yaml_parser.py | 28 -- config_file/toml_parser.py | 28 ++ config_file/utils.py | 16 ++ config_file/yaml_parser.py | 27 ++ 27 files changed, 1034 insertions(+), 1012 deletions(-) create mode 100644 config_file/abstract_parser.py create mode 100644 config_file/base_parser.py delete mode 100644 config_file/default.py create mode 100644 config_file/ini_parser.py create mode 100644 config_file/ini_parser_compat.py create mode 100644 config_file/json_parser.py create mode 100644 config_file/nested_lookup.py delete mode 100644 config_file/nested_lookup/__init__.py delete mode 100644 config_file/nested_lookup/lookup_api.py delete mode 100644 config_file/nested_lookup/nested_lookup.py create mode 100644 config_file/parse_value.py delete mode 100644 config_file/parsers/__init__.py delete mode 100644 config_file/parsers/abstract_parser.py delete mode 100644 config_file/parsers/base_parser.py delete mode 100644 config_file/parsers/ini_parser.py delete mode 100644 config_file/parsers/ini_parser_compat.py delete mode 100644 config_file/parsers/json_parser.py delete mode 100644 config_file/parsers/parse_value.py delete mode 100644 config_file/parsers/toml_parser.py delete mode 100644 config_file/parsers/yaml_parser.py create mode 100644 config_file/toml_parser.py create mode 100644 config_file/yaml_parser.py (limited to 'config_file') diff --git a/config_file/__init__.py b/config_file/__init__.py index 3825140..d78baaf 100644 --- a/config_file/__init__.py +++ b/config_file/__init__.py @@ -1,21 +1,9 @@ -from config_file.config_file import ConfigFile -from config_file.exceptions import ( - ConfigFileError, - MissingDependencyError, - MissingKeyError, - ParsingError, - UnrecognizedFileError, -) -from config_file.parsers.abstract_parser import AbstractParser +from .abstract_parser import AbstractParser +from .config_file import ConfigFile __version__ = "0.10.0" __all__ = [ "ConfigFile", "AbstractParser", - "ConfigFileError", - "ParsingError", - "UnrecognizedFileError", - "MissingDependencyError", - "MissingKeyError", ] diff --git a/config_file/abstract_parser.py b/config_file/abstract_parser.py new file mode 100644 index 0000000..ed9430b --- /dev/null +++ b/config_file/abstract_parser.py @@ -0,0 +1,117 @@ +from abc import ABC, abstractmethod +from typing import Any + + +class AbstractParser(ABC): + """Abstract Parser is the API contract for all parsers. + + Every parser in ConfigFile implements this parser, and + any user created parsers must implement this parser in + order for this package to be able to use it. + + Every key is specified in a dot (.) syntax + e.g. retrieving.some.deeply.nested.value + + If there is no dot in the key, it is assumed we are + performing an action on a top level key, which could + be an entire section. + + Args: + file_contents: The stringified version of a file. + + Attributes: + file_contents: The stringified version of a file. + """ + + @abstractmethod + def __init__(self, file_contents: str) -> None: + self.file_contents = file_contents + + @abstractmethod + def reset_internal_contents(self, file_contents: str) -> None: + """Reset the state of this parser to a new file. + + When restoring to the original file in ConfigFile, + we need a way to reset the internal contents to this + new file. + + While this may be as simple as setting the file_contents + attribute value to the passed in file_contents argument, + it likely would not be. This is because the initial + passed in file contents would likely be parsed into some + special form for the parser to work with more easily. + + Args: + file_contents: The file contents of the new file. + """ + raise NotImplementedError + + @abstractmethod + def get(self, key: str) -> Any: + """Retrieve the value of a key from the working file. + + Args: + key: The key to retrieve. + + Returns: + Whatever value the given key happens to contain. + + Raises: + MissingKeyError: If the key to retrieve does not exist. + """ + raise NotImplementedError + + @abstractmethod + def set(self, key: str, value: Any) -> None: + """Set the value of a key in the working file. + + If the key does not exist, it should be created. + + Args: + key: The key to set. + value: The value to set the key to. + """ + raise NotImplementedError + + @abstractmethod + def delete(self, key: str) -> None: + """Deletes a key from the working file. + + Args: + key: The key we would like to remove. + + Raises: + MissingKeyError: If the key to remove does not exist. + """ + raise NotImplementedError + + @abstractmethod + def stringify(self) -> str: + """Stringify the working file. + + Since the ConfigFile does not write out to disk + after every operation, this method is what is + used as the output for saving the file again. + This parser should never write out to the disk. + + Returns: + The file we are working with as a string. + """ + raise NotImplementedError + + @abstractmethod + def has(self, key: str) -> bool: + """Check whether the given key is in the parsed working file. + + This should not behave like a wild card, as in check + if the key exists anywhere in the working file. Instead, + if a key is given without a dot, it should be assumed we + are checking the top level. + + Args: + key: The key to search for. + + Returns: + True if the working file has the key. False otherwise. + """ + raise NotImplementedError diff --git a/config_file/base_parser.py b/config_file/base_parser.py new file mode 100644 index 0000000..a987a55 --- /dev/null +++ b/config_file/base_parser.py @@ -0,0 +1,139 @@ +from types import ModuleType +from typing import Any, Type + +from config_file.abstract_parser import AbstractParser +from config_file.exceptions import MissingKeyError, ParsingError +from config_file.nested_lookup import get_occurrence_of_key +from config_file.utils import split_on_dot + + +class BaseParser(AbstractParser): + def __init__( + self, file_contents: str, module: ModuleType, decode_error: Type[Exception] + ): + """The BaseParser implements the AbstractParser for us by giving it any module. + + As long as the module has a `loads` and `dumps` method, the BaseParser can use + it. Every file is then represented and worked with as a dictionary. + + Args: + file_contents: The contents of the file to parse. + + module: The module to use to parse the files. + Must have a `loads` and `dumps` method. + + decode_error: The error the module raises when the + file cannot be decoded. + + Raises: + ParsingError: If the decode_error is raised. + """ + super().__init__(file_contents) + self.__module = module + self.__decode_error = decode_error + self.parsed_content = self.parse_file_contents(module, decode_error) + + def parse_file_contents(self, module: ModuleType, decode_error: Exception) -> dict: + """Parse the file contents by running the `loads` method on the module. + + Args: + module: The module to use to parse the file. + decode_error: The error that is raised from the module if the file + cannot be decoded. + + Raises: + ParsingError: If the decode_error is raised while running `loads`. + + Returns: + The parsed file as a dictionary. + """ + try: + return module.loads(self.file_contents) + except decode_error as error: + raise ParsingError(error) + + def reset_internal_contents(self, file_contents: str) -> None: + """Reset the file contents and parsed contents of the parser. + + Args: + file_contents: The new file contents. + + Raises: + ParsingError: If we're unable to parse the new file contents. + """ + self.file_contents = file_contents + self.parsed_content = self.parse_file_contents( + self.__module, self.__decode_error + ) + + def get(self, search_key: str) -> Any: + key_causing_error = None + + try: + if "." not in search_key: + key_causing_error = search_key + return self.parsed_content[search_key] + else: + split_keys = split_on_dot(search_key) + + content_reference = self.parsed_content + for key in split_keys: + key_causing_error = key + content_reference = content_reference[key] + + except (KeyError, TypeError): + raise KeyError( + f"cannot `get` {search_key} because " + f"{key_causing_error} is not subscriptable." + ) + + return content_reference + + def set(self, key: str, value: Any) -> None: + if "." not in key: + self.parsed_content[key] = value + return + + keys = split_on_dot(key) + content_reference = self.parsed_content + for index, key in enumerate(keys): + if index == len(keys) - 1: + content_reference[key] = value + return + + try: + content_reference = content_reference[key] + except KeyError: + content_reference[key] = {} + content_reference = content_reference[key] + + def delete(self, key: str) -> None: + try: + if "." not in key: + del self.parsed_content[key] + else: + indexes = split_on_dot(key) + indexes_length = len(indexes) + + content_reference = self.parsed_content + for index, key in enumerate(indexes): + if index == indexes_length - 1: + del content_reference[key] + break + + content_reference = content_reference[key] + except (KeyError, TypeError): + raise MissingKeyError(f"The specified key '{key}' to delete was not found.") + + def stringify(self) -> str: + return self.__module.dumps(self.parsed_content) + + def has(self, search_key: str, wild: bool = False) -> bool: + if wild: + return get_occurrence_of_key(self.parsed_content, key=search_key) > 0 + + try: + self.get(search_key) + return True + except (KeyError, ParsingError): + return False diff --git a/config_file/config_file.py b/config_file/config_file.py index 3e8a806..d047560 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -3,11 +3,11 @@ from pathlib import Path from shutil import copyfile from typing import Any, Optional, Type, Union +from config_file.abstract_parser import AbstractParser from config_file.config_file_path import ConfigFilePath -from config_file.default import Default from config_file.exceptions import ParsingError -from config_file.parsers.abstract_parser import AbstractParser -from config_file.parsers.parse_value import parse_value +from config_file.parse_value import parse_value +from config_file.utils import Default class ConfigFile: diff --git a/config_file/config_file_path.py b/config_file/config_file_path.py index a40cc40..9f2b87d 100644 --- a/config_file/config_file_path.py +++ b/config_file/config_file_path.py @@ -3,12 +3,13 @@ from pathlib import Path as _Path_ from pathlib import _posix_flavour, _windows_flavour from typing import Type -from .exceptions import UnrecognizedFileError -from .parsers.abstract_parser import AbstractParser -from .parsers.ini_parser import IniParser -from .parsers.json_parser import JsonParser -from .parsers.toml_parser import TomlParser -from .parsers.yaml_parser import YamlParser +from config_file.abstract_parser import AbstractParser +from config_file.ini_parser import IniParser +from config_file.json_parser import JsonParser +from config_file.toml_parser import TomlParser +from config_file.yaml_parser import YamlParser + +from .exceptions import UnsupportedFileTypeError from .utils import split_on_dot @@ -20,7 +21,7 @@ class ConfigFilePath(_Path_): that should be use for the given file type. See https://codereview.stackexchange.com/questions/162426/subclassing-pathlib-path - for most details on subclassing the pathlib.Path object. + for more details on subclassing the pathlib.Path object. """ _flavour = _windows_flavour if os.name == "nt" else _posix_flavour @@ -57,14 +58,14 @@ class ConfigFilePath(_Path_): """Determine what parser should be used for this file. Raises: - UnrecognizedFileError: If the extension of the file + UnsupportedFileTypeError: If the extension of the file is not recognized. Returns: The instantiated parser that should be used for this file. """ if self.extension == "": - raise UnrecognizedFileError( + raise UnsupportedFileTypeError( "Tried to determine a parser to use, but the file at " f"{self} does not have an extension." ) @@ -78,7 +79,7 @@ class ConfigFilePath(_Path_): elif self.extension == "toml": return TomlParser(self.contents) else: - raise UnrecognizedFileError( + raise UnsupportedFileTypeError( f"File path at `{self}` contains an unrecognized file type." ) diff --git a/config_file/default.py b/config_file/default.py deleted file mode 100644 index 529021c..0000000 --- a/config_file/default.py +++ /dev/null @@ -1,14 +0,0 @@ -class Default: - """ - Default is used for the `default` value in ConfigFile's `get`. - - Previously, the default value for `get` was None. However, then - the user cannot have `get` return a default value of None. So this - class is used instead to get around that limitation. - """ - - def __init__(self, value): - self.value = value - - def __repr__(self): - return "Default Value: {} ({})".format(self.value, type(self.value)) diff --git a/config_file/exceptions.py b/config_file/exceptions.py index d1b466b..22ee70c 100644 --- a/config_file/exceptions.py +++ b/config_file/exceptions.py @@ -9,7 +9,7 @@ class MissingDependencyError(ConfigFileError): """ -class UnrecognizedFileError(ConfigFileError): +class UnsupportedFileTypeError(ConfigFileError): """ A missing file extension or a file type not supported. """ diff --git a/config_file/ini_parser.py b/config_file/ini_parser.py new file mode 100644 index 0000000..72974f1 --- /dev/null +++ b/config_file/ini_parser.py @@ -0,0 +1,13 @@ +import configparser + +from .base_parser import BaseParser +from .ini_parser_compat import IniParserCompatibility + + +class IniParser(BaseParser): + def __init__(self, file_contents: str): + super().__init__( + file_contents=file_contents, + module=IniParserCompatibility, + decode_error=configparser.Error, + ) diff --git a/config_file/ini_parser_compat.py b/config_file/ini_parser_compat.py new file mode 100644 index 0000000..9dfea2e --- /dev/null +++ b/config_file/ini_parser_compat.py @@ -0,0 +1,42 @@ +import configparser +from io import StringIO + + +class IniParserCompatibility: + """ + Provides compatibility with the json, yaml, + and toml API so we can use the BaseParser + easily, while still being able to leverage + the built-in configparser to parse the file. + + It is used as the `module` parameter + for BaseParser, which requires us to have a + `dumps` and `loads` method. + """ + + @staticmethod + def loads(file_contents: str) -> dict: + parser = configparser.ConfigParser() + parser.read_string(file_contents) + + return IniParserCompatibility.__create_configparser_dict(parser) + + @staticmethod + def dumps(file_contents: dict) -> str: + parser = configparser.ConfigParser() + buffer = StringIO() + + parser.read_dict(file_contents) + parser.write(buffer) + + return buffer.getvalue() + + @staticmethod + def __create_configparser_dict(parser: configparser.ConfigParser) -> dict: + result = {} + items = dict(parser.items()) + + for section in parser.sections(): + result.update({str(section): dict(items[section])}) + + return result diff --git a/config_file/json_parser.py b/config_file/json_parser.py new file mode 100644 index 0000000..5314cd7 --- /dev/null +++ b/config_file/json_parser.py @@ -0,0 +1,15 @@ +import json + +from .base_parser import BaseParser + + +class JsonParser(BaseParser): + def __init__(self, file_contents: str): + super().__init__( + file_contents=file_contents, + module=json, + decode_error=json.decoder.JSONDecodeError, + ) + + def stringify(self) -> str: + return json.dumps(self.parsed_content, indent=4) diff --git a/config_file/nested_lookup.py b/config_file/nested_lookup.py new file mode 100644 index 0000000..40da123 --- /dev/null +++ b/config_file/nested_lookup.py @@ -0,0 +1,402 @@ +import copy +import warnings +from collections import defaultdict + +__all__ = [ + "get_all_keys", + "get_occurrences_and_values", + "get_occurrence_of_value", + "get_occurrence_of_key", + "nested_update", + "nested_delete", + "nested_alter", + "nested_lookup", +] + + +values_list = [] + + +def nested_lookup(key, document, wild=False, with_keys=False): + """Lookup a key in a nested document, return a list of values""" + if with_keys: + d = defaultdict(list) + for k, v in _nested_lookup(key, document, wild=wild, with_keys=with_keys): + d[k].append(v) + return d + return list(_nested_lookup(key, document, wild=wild, with_keys=with_keys)) + + +def _is_case_insensitive_substring(a, b): + """return True if `a` is a case insensitive substring of `b`, else False""" + return str(a).lower() in str(b).lower() + + +def _nested_lookup(key, document, wild=False, with_keys=False): + """Lookup a key in a nested document, yield a value""" + if isinstance(document, list): + for d in document: + for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): + yield result + + if isinstance(document, dict): + for k, v in document.items(): + if key == k or (wild and _is_case_insensitive_substring(key, k)): + if with_keys: + yield k, v + else: + yield v + if isinstance(v, dict): + for result in _nested_lookup(key, v, wild=wild, with_keys=with_keys): + yield result + elif isinstance(v, list): + for d in v: + for result in _nested_lookup( + key, d, wild=wild, with_keys=with_keys + ): + yield result + + +def get_all_keys(dictionary): + """ + Method to get all keys from a nested dictionary as a List + Args: + dictionary: Nested dictionary + Returns: + List of keys in the dictionary + """ + result_list = [] + + def recrusion(document): + if isinstance(document, list): + for list_items in document: + recrusion(document=list_items) + elif isinstance(document, dict): + for key, value in document.items(): + result_list.append(key) + recrusion(document=value) + return + + recrusion(document=dictionary) + return result_list + + +def get_occurrence_of_key(dictionary, key): + """ + Method to get occurrence of a key in a nested dictionary + + Args: + dictionary: Nested dictionary + key: Key to search for the occurrences + Return: + Number of occurrence (Integer) + """ + return _get_occurrence(dictionary=dictionary, item="key", keyword=key) + + +def get_occurrences_and_values(items, value): + """ + Method to get occurrence of a value in a nested list of dictionary + + Args: + items: list of dictionary: Nested dictionary + value: Value to search for the occurrences + + Return: + Dict where the key is the value arg and his value is a new + dict with occurrences and values + """ + occurrences = {} + occurrence = 0 + value_list = [] + + for item in items: + occurrence_result, values = _get_occurrence_with_values( + dictionary=item, item="value", keyword=value + ) + occurrence = occurrence + occurrence_result + if occurrence_result: + value_list.extend(values) + + occurrences[value] = {"occurrences": occurrence, "values": value_list} + + return occurrences + + +def _get_occurrence_with_values(dictionary, item, keyword): + occurrence = [0] + + result_recursion = _recursion(dictionary, item, keyword, occurrence, True) + + global values_list + values_list = [] + + return occurrence[0], result_recursion + + +def get_occurrence_of_value(dictionary, value): + """ + Method to get occurrence of a value in a nested dictionary + + Args: + dictionary: Nested dictionary + value: Value to search for the occurrences + Return: + Number of occurrence (Integer) + """ + return _get_occurrence(dictionary=dictionary, item="value", keyword=value) + + +def _recursion(dictionary, item, keyword, occurrence, with_values=False): + + global values_list + + if item == "key": + if dictionary.get(keyword) is not None: + occurrence[0] += 1 + elif keyword in list(dictionary.values()): + occurrence[0] += list(dictionary.values()).count(keyword) + if with_values: + values_list.append(dictionary) + for key, value in dictionary.items(): + if isinstance(value, dict): + _recursion(value, item, keyword, occurrence, with_values) + elif isinstance(value, list): + for list_items in value: + if hasattr(list_items, "items"): + _recursion(list_items, item, keyword, occurrence, with_values) + elif list_items == keyword: + occurrence[0] += 1 if item == "value" else 0 + + if values_list: + return values_list + + +def _get_occurrence(dictionary, item, keyword): + """ + Method to get occurrence of a key or value in a nested dictionary + + Args: + dictionary: Nested dictionary + item: Mostly (key or value) + keyword: key word to find occurrence + Return: + Number of occurrence of the given keyword in the dict + """ + occurrence = [0] + _recursion(dictionary, item, keyword, occurrence) + + global values_list + values_list = [] + + return occurrence[0] + + +def nested_delete(document, key, in_place=False): + if not in_place: + document = copy.deepcopy(document) + return _nested_delete(document=document, key=key) + + +def _nested_delete(document, key): + """ + Method to delete a key->value pair from a nested document + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to delete + Return: + Returns a document that includes everything but the given key + """ + if isinstance(document, list): + for list_items in document: + _nested_delete(document=list_items, key=key) + elif isinstance(document, dict): + if document.get(key): + del document[key] + for dict_key, dict_value in document.items(): + _nested_delete(document=dict_value, key=key) + return document + + +def nested_update(document, key, value, in_place=False, treat_as_element=True): + """ + Method to update a key->value pair in a nested document + + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + + key: Key to update the value + + value: Value to set + + in_place (bool): + True: modify the dict in place; + False: create a deep copy of the dict and modify it + Defaults to False + + treat_as_element (bool): + True: if a list is provided as "value", the function trys + to match the list elements to the occurences of the key. + If the key occures more often than the provided list has + elements, the first element gets recycled. + False: the provided list is treated as one scalar value and + will be set as value to every key that matches. + Defaults to True (because of backwards portability of the package). + + Return: + Returns a document that has updated key, value pair. + """ + + # check if a list or scalar value is provided and create a list + # from the scalar value + # check the length of the list and provide it to _nested_update + if not treat_as_element and not isinstance(value, list): + raise Exception("The value must be a list when treat_as_element is False.") + elif treat_as_element: + value = [value] + + if not in_place: + document = copy.deepcopy(document) + return _nested_update(document=document, key=key, value=value) + + +def _nested_update(document, key, value): + """ + Method to update a key->value pair in a nested document. + If the number of passed values is less than the number of key matches + when scanning for updates, use last value as default. + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key (str): Key to update the value + value (list): value(s) which should be used for replacement purpouse + Return: + Returns a document that has updated key, value pair. + """ + if isinstance(document, list): + for list_items in document: + _nested_update(document=list_items, key=key, value=value) + elif isinstance(document, dict): + for dict_key, dict_value in document.items(): + if dict_key == key: + document[key] = value[0] + if len(value) > 1: + value.pop(0) + _nested_update(document=dict_value, key=key, value=value) + return document + + +def nested_alter( + document, + key, + callback_function=None, + function_parameters=None, + conversion_function=None, + wild_alter=False, + in_place=False, +): + """ + Method to alter all values of the occurences of the key "key". + The provided callback_function is used to alter the scalar values + Args: + document: Might be List of Dicts (or) Dict of Lists (or) + Dict of List of Dicts etc... + key: Key to update the value + callback_function :A callback function which alters a scalar value + HINT: You should be aware that not every element might be of + the same type, please check this in your function! + function_parameters (list): + If the callback_function has additional input arguments except + the scalar value, please specify those in this list. + conversion_function: A conversion function like str() which should be + applied to every found value before it is passed to the + "callback_function" + wild_alter: Find matching elements via wild-match by the given keys + and alter those. + HINT: Keep in mind that the wild-match might return unexpected types! + in_place (bool): + True: modify the dict in place; + False: create a deep copy of the dict and modify it + Defaults to False + Return: + Returns a document that has updated key, value pair. + """ + # check if a list or scalar value is provided and create a list from + # the scalar value + # check the length of the list and provide it to _nested_update + if isinstance(key, list): + key_len = len(key) + else: + key = [key] + key_len = len(key) + + if not in_place: + document = copy.deepcopy(document) + return _nested_alter( + document=document, + keys=key, + callback_function=callback_function, + function_parameters=function_parameters, + conversion_function=conversion_function, + wild_alter=wild_alter, + in_place=in_place, + key_len=key_len, + ) + + +def _call_callback( + value_list, callback_function, function_parameters, conversion_function +): + """ + internal helper to call the callback function + """ + return_list = [] + # loop over all values + for value in value_list: + # apply the conversion function + if conversion_function is not None: + value = conversion_function(value) + # if functions arguments are present, expand the list to variables + # via the magic operator * + if function_parameters: + trans_val = callback_function(value, *function_parameters) + else: + trans_val = callback_function(value) + # append the transformed element to the list + return_list.append(trans_val) + return return_list + + +def _nested_alter( + document, + keys, + callback_function, + function_parameters, + conversion_function, + wild_alter, + in_place, + key_len, +): + + # return data if no callback_function is provided + if callback_function is None: + warnings.warn("Please provide a callback_function to nested_alter().") + return document + + # iterate over all given keys in the list + for key in keys: + # try to find the key: + findings = nested_lookup(key, document, with_keys=True, wild=wild_alter) + for k, v in findings.items(): + trans_val = _call_callback( + v, callback_function, function_parameters, conversion_function + ) + # use the transformed value and apply the update to the key + # (dont treat the lists as elements here) + document = nested_update( + document, k, trans_val, in_place=in_place, treat_as_element=False + ) + + return document diff --git a/config_file/nested_lookup/__init__.py b/config_file/nested_lookup/__init__.py deleted file mode 100644 index 4493bdf..0000000 --- a/config_file/nested_lookup/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from .lookup_api import nested_alter, nested_delete, nested_update -from .nested_lookup import ( - get_all_keys, - get_occurrence_of_key, - get_occurrence_of_value, - get_occurrences_and_values, - nested_lookup, -) - -__all__ = [ - "get_all_keys", - "get_occurrences_and_values", - "get_occurrence_of_value", - "get_occurrence_of_key", - "nested_update", - "nested_delete", - "nested_alter", - "nested_lookup", -] diff --git a/config_file/nested_lookup/lookup_api.py b/config_file/nested_lookup/lookup_api.py deleted file mode 100644 index 886e8bf..0000000 --- a/config_file/nested_lookup/lookup_api.py +++ /dev/null @@ -1,214 +0,0 @@ -import copy -import warnings - -from .nested_lookup import nested_lookup - - -def nested_delete(document, key, in_place=False): - if not in_place: - document = copy.deepcopy(document) - return _nested_delete(document=document, key=key) - - -def _nested_delete(document, key): - """ - Method to delete a key->value pair from a nested document - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key: Key to delete - Return: - Returns a document that includes everything but the given key - """ - if isinstance(document, list): - for list_items in document: - _nested_delete(document=list_items, key=key) - elif isinstance(document, dict): - if document.get(key): - del document[key] - for dict_key, dict_value in document.items(): - _nested_delete(document=dict_value, key=key) - return document - - -def nested_update(document, key, value, in_place=False, treat_as_element=True): - """ - Method to update a key->value pair in a nested document - - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - - key: Key to update the value - - value: Value to set - - in_place (bool): - True: modify the dict in place; - False: create a deep copy of the dict and modify it - Defaults to False - - treat_as_element (bool): - True: if a list is provided as "value", the function trys - to match the list elements to the occurences of the key. - If the key occures more often than the provided list has - elements, the first element gets recycled. - False: the provided list is treated as one scalar value and - will be set as value to every key that matches. - Defaults to True (because of backwards portability of the package). - - Return: - Returns a document that has updated key, value pair. - """ - - # check if a list or scalar value is provided and create a list - # from the scalar value - # check the length of the list and provide it to _nested_update - if not treat_as_element and not isinstance(value, list): - raise Exception("The value must be a list when treat_as_element is False.") - elif treat_as_element: - value = [value] - - if not in_place: - document = copy.deepcopy(document) - return _nested_update(document=document, key=key, value=value) - - -def _nested_update(document, key, value): - """ - Method to update a key->value pair in a nested document. - If the number of passed values is less than the number of key matches - when scanning for updates, use last value as default. - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key (str): Key to update the value - value (list): value(s) which should be used for replacement purpouse - Return: - Returns a document that has updated key, value pair. - """ - if isinstance(document, list): - for list_items in document: - _nested_update(document=list_items, key=key, value=value) - elif isinstance(document, dict): - for dict_key, dict_value in document.items(): - if dict_key == key: - document[key] = value[0] - if len(value) > 1: - value.pop(0) - _nested_update(document=dict_value, key=key, value=value) - return document - - -def nested_alter( - document, - key, - callback_function=None, - function_parameters=None, - conversion_function=None, - wild_alter=False, - in_place=False, -): - """ - Method to alter all values of the occurences of the key "key". - The provided callback_function is used to alter the scalar values - Args: - document: Might be List of Dicts (or) Dict of Lists (or) - Dict of List of Dicts etc... - key: Key to update the value - callback_function :A callback function which alters a scalar value - HINT: You should be aware that not every element might be of - the same type, please check this in your function! - function_parameters (list): - If the callback_function has additional input arguments except - the scalar value, please specify those in this list. - conversion_function: A conversion function like str() which should be - applied to every found value before it is passed to the - "callback_function" - wild_alter: Find matching elements via wild-match by the given keys - and alter those. - HINT: Keep in mind that the wild-match might return unexpected types! - in_place (bool): - True: modify the dict in place; - False: create a deep copy of the dict and modify it - Defaults to False - Return: - Returns a document that has updated key, value pair. - """ - # check if a list or scalar value is provided and create a list from - # the scalar value - # check the length of the list and provide it to _nested_update - if isinstance(key, list): - key_len = len(key) - else: - key = [key] - key_len = len(key) - - if not in_place: - document = copy.deepcopy(document) - return _nested_alter( - document=document, - keys=key, - callback_function=callback_function, - function_parameters=function_parameters, - conversion_function=conversion_function, - wild_alter=wild_alter, - in_place=in_place, - key_len=key_len, - ) - - -def _call_callback( - value_list, callback_function, function_parameters, conversion_function -): - """ - internal helper to call the callback function - """ - return_list = [] - # loop over all values - for value in value_list: - # apply the conversion function - if conversion_function is not None: - value = conversion_function(value) - # if functions arguments are present, expand the list to variables - # via the magic operator * - if function_parameters: - trans_val = callback_function(value, *function_parameters) - else: - trans_val = callback_function(value) - # append the transformed element to the list - return_list.append(trans_val) - return return_list - - -def _nested_alter( - document, - keys, - callback_function, - function_parameters, - conversion_function, - wild_alter, - in_place, - key_len, -): - - # return data if no callback_function is provided - if callback_function is None: - warnings.warn("Please provide a callback_function to nested_alter().") - return document - - # iterate over all given keys in the list - for key in keys: - # try to find the key: - findings = nested_lookup(key, document, with_keys=True, wild=wild_alter) - for k, v in findings.items(): - trans_val = _call_callback( - v, callback_function, function_parameters, conversion_function - ) - # use the transformed value and apply the update to the key - # (dont treat the lists as elements here) - document = nested_update( - document, k, trans_val, in_place=in_place, treat_as_element=False - ) - - return document diff --git a/config_file/nested_lookup/nested_lookup.py b/config_file/nested_lookup/nested_lookup.py deleted file mode 100644 index 6f2e7f2..0000000 --- a/config_file/nested_lookup/nested_lookup.py +++ /dev/null @@ -1,178 +0,0 @@ -from collections import defaultdict - -values_list = [] - - -def nested_lookup(key, document, wild=False, with_keys=False): - """Lookup a key in a nested document, return a list of values""" - if with_keys: - d = defaultdict(list) - for k, v in _nested_lookup(key, document, wild=wild, with_keys=with_keys): - d[k].append(v) - return d - return list(_nested_lookup(key, document, wild=wild, with_keys=with_keys)) - - -def _is_case_insensitive_substring(a, b): - """return True if `a` is a case insensitive substring of `b`, else False""" - return str(a).lower() in str(b).lower() - - -def _nested_lookup(key, document, wild=False, with_keys=False): - """Lookup a key in a nested document, yield a value""" - if isinstance(document, list): - for d in document: - for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys): - yield result - - if isinstance(document, dict): - for k, v in document.items(): - if key == k or (wild and _is_case_insensitive_substring(key, k)): - if with_keys: - yield k, v - else: - yield v - if isinstance(v, dict): - for result in _nested_lookup(key, v, wild=wild, with_keys=with_keys): - yield result - elif isinstance(v, list): - for d in v: - for result in _nested_lookup( - key, d, wild=wild, with_keys=with_keys - ): - yield result - - -def get_all_keys(dictionary): - """ - Method to get all keys from a nested dictionary as a List - Args: - dictionary: Nested dictionary - Returns: - List of keys in the dictionary - """ - result_list = [] - - def recrusion(document): - if isinstance(document, list): - for list_items in document: - recrusion(document=list_items) - elif isinstance(document, dict): - for key, value in document.items(): - result_list.append(key) - recrusion(document=value) - return - - recrusion(document=dictionary) - return result_list - - -def get_occurrence_of_key(dictionary, key): - """ - Method to get occurrence of a key in a nested dictionary - - Args: - dictionary: Nested dictionary - key: Key to search for the occurrences - Return: - Number of occurrence (Integer) - """ - return _get_occurrence(dictionary=dictionary, item="key", keyword=key) - - -def get_occurrences_and_values(items, value): - """ - Method to get occurrence of a value in a nested list of dictionary - - Args: - items: list of dictionary: Nested dictionary - value: Value to search for the occurrences - - Return: - Dict where the key is the value arg and his value is a new - dict with occurrences and values - """ - occurrences = {} - occurrence = 0 - value_list = [] - - for item in items: - occurrence_result, values = _get_occurrence_with_values( - dictionary=item, item="value", keyword=value - ) - occurrence = occurrence + occurrence_result - if occurrence_result: - value_list.extend(values) - - occurrences[value] = {"occurrences": occurrence, "values": value_list} - - return occurrences - - -def _get_occurrence_with_values(dictionary, item, keyword): - occurrence = [0] - - result_recursion = _recursion(dictionary, item, keyword, occurrence, True) - - global values_list - values_list = [] - - return occurrence[0], result_recursion - - -def get_occurrence_of_value(dictionary, value): - """ - Method to get occurrence of a value in a nested dictionary - - Args: - dictionary: Nested dictionary - value: Value to search for the occurrences - Return: - Number of occurrence (Integer) - """ - return _get_occurrence(dictionary=dictionary, item="value", keyword=value) - - -def _recursion(dictionary, item, keyword, occurrence, with_values=False): - - global values_list - - if item == "key": - if dictionary.get(keyword) is not None: - occurrence[0] += 1 - elif keyword in list(dictionary.values()): - occurrence[0] += list(dictionary.values()).count(keyword) - if with_values: - values_list.append(dictionary) - for key, value in dictionary.items(): - if isinstance(value, dict): - _recursion(value, item, keyword, occurrence, with_values) - elif isinstance(value, list): - for list_items in value: - if hasattr(list_items, "items"): - _recursion(list_items, item, keyword, occurrence, with_values) - elif list_items == keyword: - occurrence[0] += 1 if item == "value" else 0 - - if values_list: - return values_list - - -def _get_occurrence(dictionary, item, keyword): - """ - Method to get occurrence of a key or value in a nested dictionary - - Args: - dictionary: Nested dictionary - item: Mostly (key or value) - keyword: key word to find occurrence - Return: - Number of occurrence of the given keyword in the dict - """ - occurrence = [0] - _recursion(dictionary, item, keyword, occurrence) - - global values_list - values_list = [] - - return occurrence[0] diff --git a/config_file/parse_value.py b/config_file/parse_value.py new file mode 100644 index 0000000..12032d6 --- /dev/null +++ b/config_file/parse_value.py @@ -0,0 +1,218 @@ +"""This module allows you to parse a string into its native type.""" +import ast +import re +from distutils.util import strtobool +from typing import Any + + +def parse_value(value: Any) -> Any: + """Parse a value into native type. + + parse_value('5') -> 5 (int) + parse_value('-5') -> -5 (int) + + parse_value('5.52') -> 5.52 (float) + parse_value('-53.2') -> -53.2 (float) + + parse_value('blahfafda') -> 'blahfafda' (string) + + parse_value('true') -> True (bool) + parse_value('FALSE') -> False (bool) + + parse_value(" [ [ [ [] ]] ] ") -> [[[[]]]] (list) + + Args: + value: The value to parse. + + Returns: + The parsed value, if it could be parsed + as a dictionary, list, int, float, or bool. + Otherwise, it simply returns the original + passed in value. + """ + if isinstance(value, dict): + parsed = {} + + for item in value: + parsed[item] = parse_value(value[item]) + + return parsed + + if isinstance(value, list): + parsed = [] + + for item in value: + parsed.append(parse_value(item)) + + return parsed + + if can_be_parsed_as_int(value): + return int(value.strip()) if isinstance(value, str) else int(value) + + if can_be_parsed_as_float(value): + return float(value.strip()) if isinstance(value, str) else float(value) + + if can_be_parsed_as_bool(value): + return bool(value) if isinstance(value, bool) else bool(strtobool(value)) + + if can_be_parsed_as_dict(value): + return parse_value(ast.literal_eval(value.strip())) + + if can_be_parsed_as_list(value): + return parse_value(ast.literal_eval(value.strip())) + + return value + + +def can_be_parsed_as_int(value: Any) -> bool: + """Check whether a value can be parsed as a integer. + + can_be_parsed_as_int(5) -> True + can_be_parsed_as_int("5") -> True + can_be_parsed_as_int("-5") -> True + can_be_parsed_as_int(True) -> False + + Args: + value: The value to check if it can be parsed as an integer. + + Returns: + True if we can parse the value as a integer. False otherwise. + """ + if isinstance(value, int) and not isinstance(value, bool): + return True + + if not isinstance(value, str): + return False + + value = value.strip() + + if value.startswith("-"): + return value[1:].isdigit() + + return value.isdigit() + + +def can_be_parsed_as_float(value: Any) -> bool: + """Checks whether a value can be parsed as a float. + + can_be_parsed_as_float(45) -> False + can_be_parsed_as_float(1.1) -> True + can_be_parsed_as_float("1.1") -> True + can_be_parsed_as_float("-1.1") -> True + can_be_parsed_as_float(".1") -> True + + Args: + value: The value to be parsed. + + Returns: + True if the value can be parsed as a float. False otherwise. + """ + FLOAT_REGEX = r"^\d*\.\d+$" + + if isinstance(value, float): + return True + + if not isinstance(value, str): + return False + + value = value.strip() + + if value.startswith("-"): + value = value[1:] + + return bool(re.match(FLOAT_REGEX, value)) + + +def can_be_parsed_as_bool(value: Any) -> bool: + """Checks whether a value can be parsed as a boolean. + + can_be_parsed_as_bool(True) -> True + can_be_parsed_as_bool("true") -> True + can_be_parsed_as_bool("false") -> True + can_be_parsed_as_bool("TRUE") -> True + can_be_parsed_as_bool(0) -> False + + Args: + value: The value to be parsed. + + Returns: + True if the value can be parsed as a boolean. False otherwise. + """ + if isinstance(value, bool): + return True + + if not isinstance(value, str): + return False + + value = value.lower().strip() + + return value == "true" or value == "false" + + +def can_be_parsed_as_dict(value: Any) -> bool: + """Checks whether a value can be parsed as a dictionary. + + can_be_parsed_as_dict({}) -> True + can_be_parsed_as_dict("{}") -> True + can_be_parsed_as_dict(" { } ") -> True + can_be_parsed_as_dict("{ 'hello': 5 }") -> True + can_be_parsed_as_dict([]]) -> False + can_be_parsed_as_dict(" { invalid } ") -> False + + Args: + value: The value to be parsed. + + Returns: + True if the value can be parsed as a dictionary. False otherwise. + """ + if isinstance(value, dict): + return True + + if not isinstance(value, str): + return False + + value = value.strip() + + if value[0] == "{" and value[-1] == "}": + try: + ast.literal_eval(value) + return True + except Exception: + return False + + return False + + +def can_be_parsed_as_list(value: Any) -> bool: + """Checks whether a value can be parsed as a list. + + can_be_parsed_as_list([]) -> True + can_be_parsed_as_list([1, 2]) -> True + can_be_parsed_as_list("[]") -> True + can_be_parsed_as_list(" [] ") -> True + can_be_parsed_as_list("[1, 2]") -> True + can_be_parsed_as_list({}) -> False + can_be_parsed_as_list(" [ invalid ] ") -> False + + Args: + value: The value to be parsed. + + Returns: + True if the value can be parsed as a list. False otherwise. + """ + if isinstance(value, list): + return True + + if not isinstance(value, str): + return False + + value = value.strip() + + if value[0] == "[" and value[-1] == "]": + try: + ast.literal_eval(value) + return True + except Exception: + return False + + return False diff --git a/config_file/parsers/__init__.py b/config_file/parsers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/config_file/parsers/abstract_parser.py b/config_file/parsers/abstract_parser.py deleted file mode 100644 index ed9430b..0000000 --- a/config_file/parsers/abstract_parser.py +++ /dev/null @@ -1,117 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any - - -class AbstractParser(ABC): - """Abstract Parser is the API contract for all parsers. - - Every parser in ConfigFile implements this parser, and - any user created parsers must implement this parser in - order for this package to be able to use it. - - Every key is specified in a dot (.) syntax - e.g. retrieving.some.deeply.nested.value - - If there is no dot in the key, it is assumed we are - performing an action on a top level key, which could - be an entire section. - - Args: - file_contents: The stringified version of a file. - - Attributes: - file_contents: The stringified version of a file. - """ - - @abstractmethod - def __init__(self, file_contents: str) -> None: - self.file_contents = file_contents - - @abstractmethod - def reset_internal_contents(self, file_contents: str) -> None: - """Reset the state of this parser to a new file. - - When restoring to the original file in ConfigFile, - we need a way to reset the internal contents to this - new file. - - While this may be as simple as setting the file_contents - attribute value to the passed in file_contents argument, - it likely would not be. This is because the initial - passed in file contents would likely be parsed into some - special form for the parser to work with more easily. - - Args: - file_contents: The file contents of the new file. - """ - raise NotImplementedError - - @abstractmethod - def get(self, key: str) -> Any: - """Retrieve the value of a key from the working file. - - Args: - key: The key to retrieve. - - Returns: - Whatever value the given key happens to contain. - - Raises: - MissingKeyError: If the key to retrieve does not exist. - """ - raise NotImplementedError - - @abstractmethod - def set(self, key: str, value: Any) -> None: - """Set the value of a key in the working file. - - If the key does not exist, it should be created. - - Args: - key: The key to set. - value: The value to set the key to. - """ - raise NotImplementedError - - @abstractmethod - def delete(self, key: str) -> None: - """Deletes a key from the working file. - - Args: - key: The key we would like to remove. - - Raises: - MissingKeyError: If the key to remove does not exist. - """ - raise NotImplementedError - - @abstractmethod - def stringify(self) -> str: - """Stringify the working file. - - Since the ConfigFile does not write out to disk - after every operation, this method is what is - used as the output for saving the file again. - This parser should never write out to the disk. - - Returns: - The file we are working with as a string. - """ - raise NotImplementedError - - @abstractmethod - def has(self, key: str) -> bool: - """Check whether the given key is in the parsed working file. - - This should not behave like a wild card, as in check - if the key exists anywhere in the working file. Instead, - if a key is given without a dot, it should be assumed we - are checking the top level. - - Args: - key: The key to search for. - - Returns: - True if the working file has the key. False otherwise. - """ - raise NotImplementedError diff --git a/config_file/parsers/base_parser.py b/config_file/parsers/base_parser.py deleted file mode 100644 index c2a7d71..0000000 --- a/config_file/parsers/base_parser.py +++ /dev/null @@ -1,98 +0,0 @@ -from types import ModuleType -from typing import Any - -from config_file.exceptions import MissingKeyError, ParsingError -from config_file.nested_lookup import get_occurrence_of_key -from config_file.utils import split_on_dot - -from .abstract_parser import AbstractParser - - -class BaseParser(AbstractParser): - def __init__(self, file_contents: str, module: ModuleType, decode_error: Exception): - super().__init__(file_contents) - self.__module = module - self.__decode_error = decode_error - self.parsed_content = self.parse_file_contents(module, decode_error) - - def parse_file_contents(self, module, decode_error) -> Any: - try: - return module.loads(self.file_contents) - except decode_error as error: - raise ParsingError(error) - - def reset_internal_contents(self, file_contents: str) -> None: - self.file_contents = file_contents - self.parsed_content = self.parse_file_contents( - self.__module, self.__decode_error - ) - - def get(self, search_key: str) -> Any: - if "." not in search_key: - result = self.parsed_content[search_key] - return result - else: - split_keys = split_on_dot(search_key) - - result = self.parsed_content - key_causing_error = None - try: - for key in split_keys: - key_causing_error = key - result = result[key] - except (KeyError, TypeError): - raise ParsingError( - f"Cannot `get` {search_key} because " - f"{key_causing_error} is not subscriptable." - ) - - return result - - def set(self, key: str, value: Any) -> None: - if "." not in key: - self.parsed_content[key] = value - return - - keys = split_on_dot(key) - content_reference = self.parsed_content - for index, key in enumerate(keys): - if index == len(keys) - 1: - content_reference[key] = value - return - - try: - content_reference = content_reference[key] - except KeyError: - content_reference[key] = {} - content_reference = content_reference[key] - - def delete(self, key: str) -> None: - try: - if "." not in key: - del self.parsed_content[key] - else: - indexes = split_on_dot(key) - indexes_length = len(indexes) - - content_reference = self.parsed_content - for index, key in enumerate(indexes): - if index == indexes_length - 1: - del content_reference[key] - break - - content_reference = content_reference[key] - except (KeyError, TypeError): - raise MissingKeyError(f"The specified key '{key}' to delete was not found.") - - def stringify(self) -> str: - return self.__module.dumps(self.parsed_content) - - def has(self, search_key: str, wild: bool = False) -> bool: - if wild: - return get_occurrence_of_key(self.parsed_content, key=search_key) > 0 - - try: - self.get(search_key) - return True - except (KeyError, ParsingError): - return False diff --git a/config_file/parsers/ini_parser.py b/config_file/parsers/ini_parser.py deleted file mode 100644 index 72974f1..0000000 --- a/config_file/parsers/ini_parser.py +++ /dev/null @@ -1,13 +0,0 @@ -import configparser - -from .base_parser import BaseParser -from .ini_parser_compat import IniParserCompatibility - - -class IniParser(BaseParser): - def __init__(self, file_contents: str): - super().__init__( - file_contents=file_contents, - module=IniParserCompatibility, - decode_error=configparser.Error, - ) diff --git a/config_file/parsers/ini_parser_compat.py b/config_file/parsers/ini_parser_compat.py deleted file mode 100644 index 9dfea2e..0000000 --- a/config_file/parsers/ini_parser_compat.py +++ /dev/null @@ -1,42 +0,0 @@ -import configparser -from io import StringIO - - -class IniParserCompatibility: - """ - Provides compatibility with the json, yaml, - and toml API so we can use the BaseParser - easily, while still being able to leverage - the built-in configparser to parse the file. - - It is used as the `module` parameter - for BaseParser, which requires us to have a - `dumps` and `loads` method. - """ - - @staticmethod - def loads(file_contents: str) -> dict: - parser = configparser.ConfigParser() - parser.read_string(file_contents) - - return IniParserCompatibility.__create_configparser_dict(parser) - - @staticmethod - def dumps(file_contents: dict) -> str: - parser = configparser.ConfigParser() - buffer = StringIO() - - parser.read_dict(file_contents) - parser.write(buffer) - - return buffer.getvalue() - - @staticmethod - def __create_configparser_dict(parser: configparser.ConfigParser) -> dict: - result = {} - items = dict(parser.items()) - - for section in parser.sections(): - result.update({str(section): dict(items[section])}) - - return result diff --git a/config_file/parsers/json_parser.py b/config_file/parsers/json_parser.py deleted file mode 100644 index 5314cd7..0000000 --- a/config_file/parsers/json_parser.py +++ /dev/null @@ -1,15 +0,0 @@ -import json - -from .base_parser import BaseParser - - -class JsonParser(BaseParser): - def __init__(self, file_contents: str): - super().__init__( - file_contents=file_contents, - module=json, - decode_error=json.decoder.JSONDecodeError, - ) - - def stringify(self) -> str: - return json.dumps(self.parsed_content, indent=4) diff --git a/config_file/parsers/parse_value.py b/config_file/parsers/parse_value.py deleted file mode 100644 index 12032d6..0000000 --- a/config_file/parsers/parse_value.py +++ /dev/null @@ -1,218 +0,0 @@ -"""This module allows you to parse a string into its native type.""" -import ast -import re -from distutils.util import strtobool -from typing import Any - - -def parse_value(value: Any) -> Any: - """Parse a value into native type. - - parse_value('5') -> 5 (int) - parse_value('-5') -> -5 (int) - - parse_value('5.52') -> 5.52 (float) - parse_value('-53.2') -> -53.2 (float) - - parse_value('blahfafda') -> 'blahfafda' (string) - - parse_value('true') -> True (bool) - parse_value('FALSE') -> False (bool) - - parse_value(" [ [ [ [] ]] ] ") -> [[[[]]]] (list) - - Args: - value: The value to parse. - - Returns: - The parsed value, if it could be parsed - as a dictionary, list, int, float, or bool. - Otherwise, it simply returns the original - passed in value. - """ - if isinstance(value, dict): - parsed = {} - - for item in value: - parsed[item] = parse_value(value[item]) - - return parsed - - if isinstance(value, list): - parsed = [] - - for item in value: - parsed.append(parse_value(item)) - - return parsed - - if can_be_parsed_as_int(value): - return int(value.strip()) if isinstance(value, str) else int(value) - - if can_be_parsed_as_float(value): - return float(value.strip()) if isinstance(value, str) else float(value) - - if can_be_parsed_as_bool(value): - return bool(value) if isinstance(value, bool) else bool(strtobool(value)) - - if can_be_parsed_as_dict(value): - return parse_value(ast.literal_eval(value.strip())) - - if can_be_parsed_as_list(value): - return parse_value(ast.literal_eval(value.strip())) - - return value - - -def can_be_parsed_as_int(value: Any) -> bool: - """Check whether a value can be parsed as a integer. - - can_be_parsed_as_int(5) -> True - can_be_parsed_as_int("5") -> True - can_be_parsed_as_int("-5") -> True - can_be_parsed_as_int(True) -> False - - Args: - value: The value to check if it can be parsed as an integer. - - Returns: - True if we can parse the value as a integer. False otherwise. - """ - if isinstance(value, int) and not isinstance(value, bool): - return True - - if not isinstance(value, str): - return False - - value = value.strip() - - if value.startswith("-"): - return value[1:].isdigit() - - return value.isdigit() - - -def can_be_parsed_as_float(value: Any) -> bool: - """Checks whether a value can be parsed as a float. - - can_be_parsed_as_float(45) -> False - can_be_parsed_as_float(1.1) -> True - can_be_parsed_as_float("1.1") -> True - can_be_parsed_as_float("-1.1") -> True - can_be_parsed_as_float(".1") -> True - - Args: - value: The value to be parsed. - - Returns: - True if the value can be parsed as a float. False otherwise. - """ - FLOAT_REGEX = r"^\d*\.\d+$" - - if isinstance(value, float): - return True - - if not isinstance(value, str): - return False - - value = value.strip() - - if value.startswith("-"): - value = value[1:] - - return bool(re.match(FLOAT_REGEX, value)) - - -def can_be_parsed_as_bool(value: Any) -> bool: - """Checks whether a value can be parsed as a boolean. - - can_be_parsed_as_bool(True) -> True - can_be_parsed_as_bool("true") -> True - can_be_parsed_as_bool("false") -> True - can_be_parsed_as_bool("TRUE") -> True - can_be_parsed_as_bool(0) -> False - - Args: - value: The value to be parsed. - - Returns: - True if the value can be parsed as a boolean. False otherwise. - """ - if isinstance(value, bool): - return True - - if not isinstance(value, str): - return False - - value = value.lower().strip() - - return value == "true" or value == "false" - - -def can_be_parsed_as_dict(value: Any) -> bool: - """Checks whether a value can be parsed as a dictionary. - - can_be_parsed_as_dict({}) -> True - can_be_parsed_as_dict("{}") -> True - can_be_parsed_as_dict(" { } ") -> True - can_be_parsed_as_dict("{ 'hello': 5 }") -> True - can_be_parsed_as_dict([]]) -> False - can_be_parsed_as_dict(" { invalid } ") -> False - - Args: - value: The value to be parsed. - - Returns: - True if the value can be parsed as a dictionary. False otherwise. - """ - if isinstance(value, dict): - return True - - if not isinstance(value, str): - return False - - value = value.strip() - - if value[0] == "{" and value[-1] == "}": - try: - ast.literal_eval(value) - return True - except Exception: - return False - - return False - - -def can_be_parsed_as_list(value: Any) -> bool: - """Checks whether a value can be parsed as a list. - - can_be_parsed_as_list([]) -> True - can_be_parsed_as_list([1, 2]) -> True - can_be_parsed_as_list("[]") -> True - can_be_parsed_as_list(" [] ") -> True - can_be_parsed_as_list("[1, 2]") -> True - can_be_parsed_as_list({}) -> False - can_be_parsed_as_list(" [ invalid ] ") -> False - - Args: - value: The value to be parsed. - - Returns: - True if the value can be parsed as a list. False otherwise. - """ - if isinstance(value, list): - return True - - if not isinstance(value, str): - return False - - value = value.strip() - - if value[0] == "[" and value[-1] == "]": - try: - ast.literal_eval(value) - return True - except Exception: - return False - - return False diff --git a/config_file/parsers/toml_parser.py b/config_file/parsers/toml_parser.py deleted file mode 100644 index 1c7f8a7..0000000 --- a/config_file/parsers/toml_parser.py +++ /dev/null @@ -1,28 +0,0 @@ -try: - import tomlkit - - TOML_AVAILABLE = True -except ImportError: - TOML_AVAILABLE = False - -from config_file.exceptions import MissingDependencyError - -from .base_parser import BaseParser - - -class TomlParser(BaseParser): - def __init__(self, file_contents: str): - if not TOML_AVAILABLE: - raise MissingDependencyError( - "It doesn't appear `tomlkit` is installed, but a toml " - "file was attempted to be used. Install the `toml` " - "extra first with `pip install config-file[toml]`." - ) - - super().__init__( - file_contents=file_contents, - module=tomlkit, - # tomlkit doesn't expose it's exceptions, but - # it's parsing error is a subclass of ValueError - decode_error=ValueError, - ) diff --git a/config_file/parsers/yaml_parser.py b/config_file/parsers/yaml_parser.py deleted file mode 100644 index babe873..0000000 --- a/config_file/parsers/yaml_parser.py +++ /dev/null @@ -1,28 +0,0 @@ -try: - from ruamel.yaml import YAML, YAMLError, round_trip_dump, round_trip_load - - YAML_AVAILABLE = True -except ImportError: - YAML_AVAILABLE = False - -from config_file.exceptions import MissingDependencyError - -from .base_parser import BaseParser - - -class YamlParser(BaseParser): - def __init__(self, file_contents: str): - if not YAML_AVAILABLE: - raise MissingDependencyError( - "It doesn't appear `ruamel.yaml` is installed, but a yaml " - "file was attempted to be used. Install the `yaml` " - "extra first with `pip install config-file[yaml]`." - ) - - yaml = YAML(typ="safe") - yaml.loads = round_trip_load - yaml.dumps = round_trip_dump - - super().__init__( - file_contents=file_contents, module=yaml, decode_error=YAMLError - ) diff --git a/config_file/toml_parser.py b/config_file/toml_parser.py new file mode 100644 index 0000000..1c7f8a7 --- /dev/null +++ b/config_file/toml_parser.py @@ -0,0 +1,28 @@ +try: + import tomlkit + + TOML_AVAILABLE = True +except ImportError: + TOML_AVAILABLE = False + +from config_file.exceptions import MissingDependencyError + +from .base_parser import BaseParser + + +class TomlParser(BaseParser): + def __init__(self, file_contents: str): + if not TOML_AVAILABLE: + raise MissingDependencyError( + "It doesn't appear `tomlkit` is installed, but a toml " + "file was attempted to be used. Install the `toml` " + "extra first with `pip install config-file[toml]`." + ) + + super().__init__( + file_contents=file_contents, + module=tomlkit, + # tomlkit doesn't expose it's exceptions, but + # it's parsing error is a subclass of ValueError + decode_error=ValueError, + ) diff --git a/config_file/utils.py b/config_file/utils.py index c6c3d24..3554c0e 100644 --- a/config_file/utils.py +++ b/config_file/utils.py @@ -20,3 +20,19 @@ def split_on_dot(line: Union[str, Path], only_last_dot=False) -> List[str]: raise ValueError(f"The given string does not have a dot to split on: {line}") return line.rsplit(".", 1) if only_last_dot else line.split(".") + + +class Default: + """ + Default is used for the `default` value in ConfigFile's `get`. + + Previously, the default value for `get` was None. However, then + the user cannot have `get` return a default value of None. So this + class is used instead to get around that limitation. + """ + + def __init__(self, value): + self.value = value + + def __repr__(self): + return "Default Value: {} ({})".format(self.value, type(self.value)) diff --git a/config_file/yaml_parser.py b/config_file/yaml_parser.py new file mode 100644 index 0000000..fb42e08 --- /dev/null +++ b/config_file/yaml_parser.py @@ -0,0 +1,27 @@ +try: + from ruamel.yaml import YAML, YAMLError, round_trip_dump, round_trip_load + + YAML_AVAILABLE = True +except ImportError: + YAML_AVAILABLE = False + +from .base_parser import BaseParser +from .exceptions import MissingDependencyError + + +class YamlParser(BaseParser): + def __init__(self, file_contents: str): + if not YAML_AVAILABLE: + raise MissingDependencyError( + "It doesn't appear `ruamel.yaml` is installed, but a yaml " + "file was attempted to be used. Install the `yaml` " + "extra first with `pip install config-file[yaml]`." + ) + + yaml = YAML(typ="safe") + yaml.loads = round_trip_load + yaml.dumps = round_trip_dump + + super().__init__( + file_contents=file_contents, module=yaml, decode_error=YAMLError + ) -- cgit v1.2.3 From d9a2ade40e72176449433782957bf503d69cc1d8 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Wed, 5 Aug 2020 20:59:42 -0500 Subject: Remove custom parser and add ability to index ConfigFile --- config_file/config_file.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index d047560..0e67794 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -1,9 +1,7 @@ -import inspect from pathlib import Path from shutil import copyfile -from typing import Any, Optional, Type, Union +from typing import Any, Union -from config_file.abstract_parser import AbstractParser from config_file.config_file_path import ConfigFilePath from config_file.exceptions import ParsingError from config_file.parse_value import parse_value @@ -11,9 +9,7 @@ from config_file.utils import Default class ConfigFile: - def __init__( - self, file_path: Union[str, Path], parser: Optional[Type[AbstractParser]] = None - ) -> None: + def __init__(self, file_path: Union[str, Path]) -> None: """ Stores the config file path and expands it if needed, reads in the file contents, and determines what parser should be used for @@ -21,7 +17,6 @@ class ConfigFile: Args: file_path: The path to your configuration file. - parser: A custom parser you'd like used for your config file. Raises: ValueError: If the specified file path does not have an extension @@ -29,11 +24,7 @@ class ConfigFile: FileNotFoundError: If the specified file path does not exist. """ self.__path = ConfigFilePath(file_path).validate() - - if parser is not None and not inspect.isabstract(parser): - self.__parser = parser(self.__path.contents) - else: - self.__parser = self.__path.parser + self.__parser = self.__path.parser @property def file_path(self) -> Path: @@ -43,6 +34,15 @@ class ConfigFile: def original_file_path(self) -> Path: return Path(self.__path.original_path) + def __getitem__(self, key: str): + return self.__parser.parsed_content[key] + + def __setitem__(self, key: str, value: Any): + self.__parser.parsed_content[key] = value + + def __delitem__(self, key: str): + del self.__parser.parsed_content[key] + def get( self, key: str, -- cgit v1.2.3 From fb34cb2b000e425c152bc7d4de5c5227bed138e4 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Wed, 5 Aug 2020 21:15:47 -0500 Subject: Remove AbstractParser from __init__ --- config_file/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'config_file') diff --git a/config_file/__init__.py b/config_file/__init__.py index d78baaf..85d07ad 100644 --- a/config_file/__init__.py +++ b/config_file/__init__.py @@ -1,9 +1,5 @@ -from .abstract_parser import AbstractParser from .config_file import ConfigFile __version__ = "0.10.0" -__all__ = [ - "ConfigFile", - "AbstractParser", -] +__all__ = ["ConfigFile"] -- cgit v1.2.3 From 1fdde7b696674546220b85bb3ffd2994e93fec3a Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Wed, 5 Aug 2020 21:27:26 -0500 Subject: Add `wild` to AbstractParser --- config_file/abstract_parser.py | 3 ++- config_file/config_file.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'config_file') diff --git a/config_file/abstract_parser.py b/config_file/abstract_parser.py index ed9430b..ab5d205 100644 --- a/config_file/abstract_parser.py +++ b/config_file/abstract_parser.py @@ -100,7 +100,7 @@ class AbstractParser(ABC): raise NotImplementedError @abstractmethod - def has(self, key: str) -> bool: + def has(self, key: str, wild: bool = False) -> bool: """Check whether the given key is in the parsed working file. This should not behave like a wild card, as in check @@ -110,6 +110,7 @@ class AbstractParser(ABC): Args: key: The key to search for. + wild: Optionally search for any occurrence of key in the file. Returns: True if the working file has the key. False otherwise. diff --git a/config_file/config_file.py b/config_file/config_file.py index 0e67794..d66cc48 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -137,7 +137,7 @@ class ConfigFile: wild: Whether or not to search everywhere. Without `wild`, a single word `key` without a `.` - will look at the outer most heirarchy of the file for it. + will look at the outer most hierarchy of the file for it. With `wild`, that single word `key` will be searched for throughout the entire file. -- cgit v1.2.3 From 19d23dc13636b30f6282e8b3f02b659bcdb2bb11 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Wed, 5 Aug 2020 21:48:08 -0500 Subject: Remove MissingDependencyError in favor of ImportError --- config_file/config_file_path.py | 2 +- config_file/exceptions.py | 7 ------- config_file/toml_parser.py | 3 +-- config_file/yaml_parser.py | 3 +-- 4 files changed, 3 insertions(+), 12 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file_path.py b/config_file/config_file_path.py index 9f2b87d..42466d9 100644 --- a/config_file/config_file_path.py +++ b/config_file/config_file_path.py @@ -102,7 +102,7 @@ class ConfigFilePath(_Path_): ValueError: If the path leads to a directory. Returns: - This ConfigFilePath object + A ConfigFilePath object with any ~'s expanded. """ if len(self.parts) >= 1 and self.parts[0] == "~": self = ConfigFilePath(self.expanduser()) diff --git a/config_file/exceptions.py b/config_file/exceptions.py index 22ee70c..a8b8e5c 100644 --- a/config_file/exceptions.py +++ b/config_file/exceptions.py @@ -2,13 +2,6 @@ class ConfigFileError(Exception): """A generic error from the package.""" -class MissingDependencyError(ConfigFileError): - """ - yaml or toml dependencies were not installed, - but the parsers for them were attempted to be used. - """ - - class UnsupportedFileTypeError(ConfigFileError): """ A missing file extension or a file type not supported. diff --git a/config_file/toml_parser.py b/config_file/toml_parser.py index 1c7f8a7..fc12ab3 100644 --- a/config_file/toml_parser.py +++ b/config_file/toml_parser.py @@ -5,7 +5,6 @@ try: except ImportError: TOML_AVAILABLE = False -from config_file.exceptions import MissingDependencyError from .base_parser import BaseParser @@ -13,7 +12,7 @@ from .base_parser import BaseParser class TomlParser(BaseParser): def __init__(self, file_contents: str): if not TOML_AVAILABLE: - raise MissingDependencyError( + raise ImportError( "It doesn't appear `tomlkit` is installed, but a toml " "file was attempted to be used. Install the `toml` " "extra first with `pip install config-file[toml]`." diff --git a/config_file/yaml_parser.py b/config_file/yaml_parser.py index fb42e08..938b0d7 100644 --- a/config_file/yaml_parser.py +++ b/config_file/yaml_parser.py @@ -6,13 +6,12 @@ except ImportError: YAML_AVAILABLE = False from .base_parser import BaseParser -from .exceptions import MissingDependencyError class YamlParser(BaseParser): def __init__(self, file_contents: str): if not YAML_AVAILABLE: - raise MissingDependencyError( + raise ImportError( "It doesn't appear `ruamel.yaml` is installed, but a yaml " "file was attempted to be used. Install the `yaml` " "extra first with `pip install config-file[yaml]`." -- cgit v1.2.3 From d2f3161dac2313e4109ba2159fdc0d43e50506f8 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Thu, 6 Aug 2020 09:43:44 -0500 Subject: Test extras parsers & move imports inside those parsers --- config_file/toml_parser.py | 25 ++++++++----------------- config_file/yaml_parser.py | 27 +++++++++++---------------- 2 files changed, 19 insertions(+), 33 deletions(-) (limited to 'config_file') diff --git a/config_file/toml_parser.py b/config_file/toml_parser.py index fc12ab3..d3cf226 100644 --- a/config_file/toml_parser.py +++ b/config_file/toml_parser.py @@ -1,27 +1,18 @@ -try: - import tomlkit - - TOML_AVAILABLE = True -except ImportError: - TOML_AVAILABLE = False - - from .base_parser import BaseParser class TomlParser(BaseParser): def __init__(self, file_contents: str): - if not TOML_AVAILABLE: + try: + import tomlkit + from tomlkit.exceptions import ParseError + + super().__init__( + file_contents=file_contents, module=tomlkit, decode_error=ParseError, + ) + except ImportError: raise ImportError( "It doesn't appear `tomlkit` is installed, but a toml " "file was attempted to be used. Install the `toml` " "extra first with `pip install config-file[toml]`." ) - - super().__init__( - file_contents=file_contents, - module=tomlkit, - # tomlkit doesn't expose it's exceptions, but - # it's parsing error is a subclass of ValueError - decode_error=ValueError, - ) diff --git a/config_file/yaml_parser.py b/config_file/yaml_parser.py index 938b0d7..8d88881 100644 --- a/config_file/yaml_parser.py +++ b/config_file/yaml_parser.py @@ -1,26 +1,21 @@ -try: - from ruamel.yaml import YAML, YAMLError, round_trip_dump, round_trip_load - - YAML_AVAILABLE = True -except ImportError: - YAML_AVAILABLE = False - from .base_parser import BaseParser class YamlParser(BaseParser): def __init__(self, file_contents: str): - if not YAML_AVAILABLE: + try: + from ruamel.yaml import YAML, YAMLError, round_trip_dump, round_trip_load + + yaml = YAML(typ="safe") + yaml.loads = round_trip_load + yaml.dumps = round_trip_dump + + super().__init__( + file_contents=file_contents, module=yaml, decode_error=YAMLError + ) + except ImportError: raise ImportError( "It doesn't appear `ruamel.yaml` is installed, but a yaml " "file was attempted to be used. Install the `yaml` " "extra first with `pip install config-file[yaml]`." ) - - yaml = YAML(typ="safe") - yaml.loads = round_trip_load - yaml.dumps = round_trip_dump - - super().__init__( - file_contents=file_contents, module=yaml, decode_error=YAMLError - ) -- cgit v1.2.3 From 836cefe45633331d0e5692295174bf3181a5e365 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Thu, 6 Aug 2020 09:47:35 -0500 Subject: Change MissingKeyErrors to KeyError --- config_file/abstract_parser.py | 4 ++-- config_file/base_parser.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'config_file') diff --git a/config_file/abstract_parser.py b/config_file/abstract_parser.py index ab5d205..689fece 100644 --- a/config_file/abstract_parser.py +++ b/config_file/abstract_parser.py @@ -57,7 +57,7 @@ class AbstractParser(ABC): Whatever value the given key happens to contain. Raises: - MissingKeyError: If the key to retrieve does not exist. + KeyError: If the key to retrieve does not exist. """ raise NotImplementedError @@ -81,7 +81,7 @@ class AbstractParser(ABC): key: The key we would like to remove. Raises: - MissingKeyError: If the key to remove does not exist. + KeyError: If the key to remove does not exist. """ raise NotImplementedError diff --git a/config_file/base_parser.py b/config_file/base_parser.py index a987a55..a440dcb 100644 --- a/config_file/base_parser.py +++ b/config_file/base_parser.py @@ -2,7 +2,7 @@ from types import ModuleType from typing import Any, Type from config_file.abstract_parser import AbstractParser -from config_file.exceptions import MissingKeyError, ParsingError +from config_file.exceptions import ParsingError from config_file.nested_lookup import get_occurrence_of_key from config_file.utils import split_on_dot @@ -123,7 +123,7 @@ class BaseParser(AbstractParser): content_reference = content_reference[key] except (KeyError, TypeError): - raise MissingKeyError(f"The specified key '{key}' to delete was not found.") + raise KeyError(f"The specified key '{key}' to delete was not found.") def stringify(self) -> str: return self.__module.dumps(self.parsed_content) -- cgit v1.2.3 From 05876de8dbb429de461832fc9106fd6514a5e97a Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Fri, 7 Aug 2020 08:47:57 -0500 Subject: Add ParsingError to __init__ --- config_file/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'config_file') diff --git a/config_file/__init__.py b/config_file/__init__.py index 85d07ad..853da2a 100644 --- a/config_file/__init__.py +++ b/config_file/__init__.py @@ -1,5 +1,6 @@ from .config_file import ConfigFile +from .exceptions import ParsingError __version__ = "0.10.0" -__all__ = ["ConfigFile"] +__all__ = ["ConfigFile", "ParsingError"] -- cgit v1.2.3 From c24f1741a91950480fd169a6583e6c2b675ee08e Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Fri, 7 Aug 2020 08:49:03 -0500 Subject: Change UnsupportedFileTypeError to ValueError --- config_file/config_file_path.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file_path.py b/config_file/config_file_path.py index 42466d9..11a5daa 100644 --- a/config_file/config_file_path.py +++ b/config_file/config_file_path.py @@ -9,7 +9,6 @@ from config_file.json_parser import JsonParser from config_file.toml_parser import TomlParser from config_file.yaml_parser import YamlParser -from .exceptions import UnsupportedFileTypeError from .utils import split_on_dot @@ -58,14 +57,13 @@ class ConfigFilePath(_Path_): """Determine what parser should be used for this file. Raises: - UnsupportedFileTypeError: If the extension of the file - is not recognized. + ValueError: If the extension of the file is not recognized. Returns: The instantiated parser that should be used for this file. """ if self.extension == "": - raise UnsupportedFileTypeError( + raise ValueError( "Tried to determine a parser to use, but the file at " f"{self} does not have an extension." ) @@ -79,7 +77,7 @@ class ConfigFilePath(_Path_): elif self.extension == "toml": return TomlParser(self.contents) else: - raise UnsupportedFileTypeError( + raise ValueError( f"File path at `{self}` contains an unrecognized file type." ) -- cgit v1.2.3 From 32826f7d510ff4c469b2fa67d232325224a368eb Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Fri, 7 Aug 2020 08:49:14 -0500 Subject: Add type hint to split_on_dot --- config_file/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'config_file') diff --git a/config_file/utils.py b/config_file/utils.py index 3554c0e..cbbf7a7 100644 --- a/config_file/utils.py +++ b/config_file/utils.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import List, Union -def split_on_dot(line: Union[str, Path], only_last_dot=False) -> List[str]: +def split_on_dot(line: Union[str, Path], only_last_dot: bool = False) -> List[str]: """ Split a string on the dot character (.). -- cgit v1.2.3 From 08a0725f7d4c34ef073110c30117ae520d081a01 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Fri, 7 Aug 2020 08:50:06 -0500 Subject: Remove all custom exceptions except ParsingError --- config_file/exceptions.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) (limited to 'config_file') diff --git a/config_file/exceptions.py b/config_file/exceptions.py index a8b8e5c..f68b7e8 100644 --- a/config_file/exceptions.py +++ b/config_file/exceptions.py @@ -1,19 +1,9 @@ -class ConfigFileError(Exception): - """A generic error from the package.""" - - -class UnsupportedFileTypeError(ConfigFileError): +class ParsingError(ValueError): """ - A missing file extension or a file type not supported. - """ - + Unable to parse the configuration file. -class ParsingError(ConfigFileError): - """Unable to parse the configuration file.""" - - -class MissingKeyError(ParsingError): - """ - A key that was specified (to retrieve, delete, etc.) - is not in the file. + Since every module that parses the different + files has it's own decode error, this standardizes + it for the package to only raise a ParsingError + if a error occurs in those instances. """ -- cgit v1.2.3 From 08d7c88cae495ac8904be2367eb0d223e16ac8e8 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Fri, 7 Aug 2020 09:52:11 -0500 Subject: Fix invalid reference to MissingKeyError --- config_file/config_file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index d66cc48..2ce61ef 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -69,6 +69,7 @@ class ConfigFile: The value of the key. Raises: + KeyError: If a key is attempted to be retrieved that does not exist. ValueError: If the value is not able to be coerced into return_type. """ try: @@ -110,7 +111,7 @@ class ConfigFile: key: The section, sub-section, or key to delete. Raises: - MissingKeyError: If a key is attempted to be deleted that + KeyError: If a key is attempted to be deleted that does not exist. """ self.__parser.delete(key) -- cgit v1.2.3 From 16620abbbc92c8fa3999fb55e5d45d6397c328d6 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Fri, 7 Aug 2020 10:26:09 -0500 Subject: Fix indentation of docstring --- config_file/nested_lookup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'config_file') diff --git a/config_file/nested_lookup.py b/config_file/nested_lookup.py index 40da123..7b13d83 100644 --- a/config_file/nested_lookup.py +++ b/config_file/nested_lookup.py @@ -59,11 +59,11 @@ def _nested_lookup(key, document, wild=False, with_keys=False): def get_all_keys(dictionary): """ - Method to get all keys from a nested dictionary as a List - Args: - dictionary: Nested dictionary - Returns: - List of keys in the dictionary + Method to get all keys from a nested dictionary as a List + Args: + dictionary: Nested dictionary + Returns: + List of keys in the dictionary """ result_list = [] -- cgit v1.2.3 From e529b2be912925e91be6466dfd926610012ba58c Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Fri, 7 Aug 2020 19:40:38 -0500 Subject: Change file_path to path and original_file_path to original_path --- config_file/config_file.py | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index 2ce61ef..8cde5cb 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -27,11 +27,11 @@ class ConfigFile: self.__parser = self.__path.parser @property - def file_path(self) -> Path: + def path(self) -> Path: return Path(self.__path) @property - def original_file_path(self) -> Path: + def original_path(self) -> Path: return Path(self.__path.original_path) def __getitem__(self, key: str): @@ -148,9 +148,7 @@ class ConfigFile: """ return self.__parser.has(key, wild=wild) - def restore_original( - self, original_file_path: Union[str, Path, None] = None - ) -> None: + def restore_original(self, original_path: Union[str, Path, None] = None) -> None: """Restores the original the configuration file. The current one is deleted and the original is copied back @@ -158,32 +156,25 @@ class ConfigFile: new file. Args: - original_file_path: The original file to reset to. + original_path: The original file to reset to. - If this is not provided, it is calculated for you. - e.g. if your configuration file is named config.json, - this will look for aa config.original.json in the same - directory. + Defaults to the original_path property if it is not + provided. Raises: FileNotFoundError: If the original configuration file (whether - calculated or passed in) does not exist. + calculated or passed in) does not exist or if the current + configuration path is passed in as the original_path (since + it is deleted before the original file is copied over). OSError: If the current configuration path is not writable. - - SameFileError: If current configuration path and the passed in - original_file_path are the same file. """ - if original_file_path and not ConfigFilePath(original_file_path).exists(): - raise FileNotFoundError( - f"The {original_file_path} file to restore to does not exist." - ) - else: - original_file_path = self.__path.original_path - original_file_path = original_file_path.validate() + original_path = ConfigFilePath( + original_path if original_path else self.original_path + ).validate() self.__path.unlink() - copyfile(original_file_path, self.__path) + copyfile(original_path, self.__path) self.__parser.reset_internal_contents(self.__path.contents) def save(self) -> None: -- cgit v1.2.3 From a0cdb32e3f8ae69bd1163e2d44e3d731d9e8594e Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Fri, 7 Aug 2020 20:15:05 -0500 Subject: Fix ConfigFile default --- config_file/config_file.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'config_file') diff --git a/config_file/config_file.py b/config_file/config_file.py index 8cde5cb..2a8d9ae 100644 --- a/config_file/config_file.py +++ b/config_file/config_file.py @@ -3,7 +3,6 @@ from shutil import copyfile from typing import Any, Union from config_file.config_file_path import ConfigFilePath -from config_file.exceptions import ParsingError from config_file.parse_value import parse_value from config_file.utils import Default @@ -74,11 +73,11 @@ class ConfigFile: """ try: key_value = self.__parser.get(key) - except ParsingError as error: - if default is not type(Default): - key_value = default + except KeyError as error: + if isinstance(default, Default) and default.value is None: + raise KeyError(error) else: - raise ParsingError(error) + key_value = default if parse_types: key_value = parse_value(key_value) -- cgit v1.2.3