aboutsummaryrefslogtreecommitdiffstats
path: root/config_file/nested_lookup.py
blob: c67109c087a01aa58b94e6760e27e5f8942e9757 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
import copy
import warnings
from collections import defaultdict

__all__ = [
    "get_all_keys",
    "get_occurrences_and_values",
    "get_occurrence_of_value",
    "get_occurrence_of_key",
    "nested_update",
    "nested_delete",
    "nested_alter",
    "nested_lookup",
]


# TODO: get rid of this global values_list
values_list = []


def nested_lookup(key, document, wild=False, with_keys=False):
    """Lookup a key in a nested document, return a list of values"""
    if with_keys:
        d = defaultdict(list)
        for k, v in _nested_lookup(key, document, wild=wild, with_keys=with_keys):
            d[k].append(v)
        return d
    return list(_nested_lookup(key, document, wild=wild, with_keys=with_keys))


def _is_case_insensitive_substring(a, b):
    """return True if `a` is a case insensitive substring of `b`, else False"""
    return str(a).lower() in str(b).lower()


def _nested_lookup(key, document, wild=False, with_keys=False):
    """Lookup a key in a nested document, yield a value"""
    if isinstance(document, list):
        for d in document:
            for result in _nested_lookup(key, d, wild=wild, with_keys=with_keys):
                yield result

    if isinstance(document, dict):
        for k, v in document.items():
            if key == k or (wild and _is_case_insensitive_substring(key, k)):
                if with_keys:
                    yield k, v
                else:
                    yield v
            if isinstance(v, dict):
                for result in _nested_lookup(key, v, wild=wild, with_keys=with_keys):
                    yield result
            elif isinstance(v, list):
                for d in v:
                    for result in _nested_lookup(
                        key, d, wild=wild, with_keys=with_keys
                    ):
                        yield result


def get_all_keys(dictionary):
    """
    Method to get all keys from a nested dictionary as a List
    Args:
        dictionary: Nested dictionary
    Returns:
        List of keys in the dictionary
    """
    result_list = []

    def recrusion(document):
        if isinstance(document, list):
            for list_items in document:
                recrusion(document=list_items)
        elif isinstance(document, dict):
            for key, value in document.items():
                result_list.append(key)
                recrusion(document=value)
        return

    recrusion(document=dictionary)
    return result_list


def get_occurrence_of_key(dictionary, key):
    """
    Method to get occurrence of a key in a nested dictionary

    Args:
        dictionary: Nested dictionary
        key: Key to search for the occurrences
    Return:
        Number of occurrence (Integer)
    """
    return _get_occurrence(dictionary=dictionary, item="key", keyword=key)


def get_occurrences_and_values(items, value):
    """
    Method to get occurrence of a value in a nested list of dictionary

    Args:
        items: list of dictionary: Nested dictionary
        value: Value to search for the occurrences

    Return:
        Dict where the key is the value arg and his value is a new
        dict with occurrences and values
    """
    occurrences = {}
    occurrence = 0
    value_list = []

    for item in items:
        occurrence_result, values = _get_occurrence_with_values(
            dictionary=item, item="value", keyword=value
        )
        occurrence = occurrence + occurrence_result
        if occurrence_result:
            value_list.extend(values)

    occurrences[value] = {"occurrences": occurrence, "values": value_list}

    return occurrences


def _get_occurrence_with_values(dictionary, item, keyword):
    occurrence = [0]

    result_recursion = _recursion(dictionary, item, keyword, occurrence, True)

    global values_list
    values_list = []

    return occurrence[0], result_recursion


def get_occurrence_of_value(dictionary, value):
    """
    Method to get occurrence of a value in a nested dictionary

    Args:
        dictionary: Nested dictionary
        value: Value to search for the occurrences
    Return:
        Number of occurrence (Integer)
    """
    return _get_occurrence(dictionary=dictionary, item="value", keyword=value)


def _recursion(dictionary, item, keyword, occurrence, with_values=False):
    global values_list

    if item == "key":
        if dictionary.get(keyword) is not None:
            occurrence[0] += 1
    elif keyword in list(dictionary.values()):
        occurrence[0] += list(dictionary.values()).count(keyword)
        if with_values:
            values_list.append(dictionary)
    for key, value in dictionary.items():
        if isinstance(value, dict):
            _recursion(value, item, keyword, occurrence, with_values)
        elif isinstance(value, list):
            for list_items in value:
                if hasattr(list_items, "items"):
                    _recursion(list_items, item, keyword, occurrence, with_values)
                elif list_items == keyword:
                    occurrence[0] += 1 if item == "value" else 0

    if values_list:
        return values_list


def _get_occurrence(dictionary, item, keyword):
    """
    Method to get occurrence of a key or value in a nested dictionary

    Args:
        dictionary: Nested dictionary
        item: Mostly (key or value)
        keyword: key word to find occurrence
    Return:
        Number of occurrence of the given keyword in the dict
    """
    occurrence = [0]
    _recursion(dictionary, item, keyword, occurrence)

    global values_list
    values_list = []

    return occurrence[0]


def nested_delete(document, key, in_place=False):
    if not in_place:
        document = copy.deepcopy(document)
    return _nested_delete(document=document, key=key)


def _nested_delete(document, key):
    """
    Method to delete a key->value pair from a nested document
    Args:
        document: Might be List of Dicts (or) Dict of Lists (or)
         Dict of List of Dicts etc...
        key: Key to delete
    Return:
        Returns a document that includes everything but the given key
    """
    if isinstance(document, list):
        for list_items in document:
            _nested_delete(document=list_items, key=key)
    elif isinstance(document, dict):
        if document.get(key):
            del document[key]
        for dict_key, dict_value in document.items():
            _nested_delete(document=dict_value, key=key)
    return document


def nested_update(document, key, value, in_place=False, treat_as_element=True):
    """
    Method to update a key->value pair in a nested document

    Args:
        document: Might be List of Dicts (or) Dict of Lists (or)
            Dict of List of Dicts etc...

        key: Key to update the value

        value: Value to set

        in_place (bool):
            True: modify the dict in place;
            False: create a deep copy of the dict and modify it
            Defaults to False

        treat_as_element (bool):
            True: if a list is provided as "value", the function trys
                to match the list elements to the occurences of the key.
                If the key occures more often than the provided list has
                elements, the first element gets recycled.
            False: the provided list is treated as one scalar value and
                will be set as value to every key that matches.
            Defaults to True (because of backwards portability of the package).

    Return:
        Returns a document that has updated key, value pair.
    """

    # check if a list or scalar value is provided and create a list
    # from the scalar value
    # check the length of the list and provide it to _nested_update
    if not treat_as_element and not isinstance(value, list):
        raise Exception("The value must be a list when treat_as_element is False.")
    elif treat_as_element:
        value = [value]

    if not in_place:
        document = copy.deepcopy(document)
    return _nested_update(document=document, key=key, value=value)


def _nested_update(document, key, value):
    """
    Method to update a key->value pair in a nested document.
    If the number of passed values is less than the number of key matches
    when scanning for updates, use last value as default.
    Args:
        document: Might be List of Dicts (or) Dict of Lists (or)
            Dict of List of Dicts etc...
        key (str): Key to update the value
        value (list): value(s) which should be used for replacement purpouse
    Return:
        Returns a document that has updated key, value pair.
    """
    if isinstance(document, list):
        for list_items in document:
            _nested_update(document=list_items, key=key, value=value)
    elif isinstance(document, dict):
        for dict_key, dict_value in document.items():
            if dict_key == key:
                document[key] = value[0]
                if len(value) > 1:
                    value.pop(0)
            _nested_update(document=dict_value, key=key, value=value)
    return document


def nested_alter(
    document,
    key,
    callback_function=None,
    function_parameters=None,
    conversion_function=None,
    wild_alter=False,
    in_place=False,
):
    """
    Method to alter all values of the occurences of the key "key".
    The provided callback_function is used to alter the scalar values
    Args:
        document: Might be List of Dicts (or) Dict of Lists (or)
            Dict of List of Dicts etc...
        key: Key to update the value
        callback_function :A callback function which alters a scalar value
            HINT: You should be aware that not every element might be of
            the same type, please check this in your function!
        function_parameters (list):
            If the callback_function has additional input arguments except
            the scalar value, please specify those in this list.
        conversion_function: A conversion function like str() which should be
            applied to every found value before it is passed to the
            "callback_function"
        wild_alter: Find matching elements via wild-match by the given keys
            and alter those.
        HINT: Keep in mind that the wild-match might return unexpected types!
        in_place (bool):
            True: modify the dict in place;
            False: create a deep copy of the dict and modify it
            Defaults to False
    Return:
        Returns a document that has updated key, value pair.
    """
    # check if a list or scalar value is provided and create a list from
    # the scalar value
    # check the length of the list and provide it to _nested_update
    if isinstance(key, list):
        key_len = len(key)
    else:
        key = [key]
        key_len = len(key)

    if not in_place:
        document = copy.deepcopy(document)
    return _nested_alter(
        document=document,
        keys=key,
        callback_function=callback_function,
        function_parameters=function_parameters,
        conversion_function=conversion_function,
        wild_alter=wild_alter,
        in_place=in_place,
        key_len=key_len,
    )


def _call_callback(
    value_list, callback_function, function_parameters, conversion_function
):
    """
    internal helper to call the callback function
    """
    return_list = []
    # loop over all values
    for value in value_list:
        # apply the conversion function
        if conversion_function is not None:
            value = conversion_function(value)
        # if functions arguments are present, expand the list to variables
        # via the magic operator *
        if function_parameters:
            trans_val = callback_function(value, *function_parameters)
        else:
            trans_val = callback_function(value)
        # append the transformed element to the list
        return_list.append(trans_val)
    return return_list


def _nested_alter(
    document,
    keys,
    callback_function,
    function_parameters,
    conversion_function,
    wild_alter,
    in_place,
    key_len,
):
    # return data if no callback_function is provided
    if callback_function is None:
        warnings.warn("Please provide a callback_function to nested_alter().")
        return document

    # iterate over all given keys in the list
    for key in keys:
        # try to find the key:
        findings = nested_lookup(key, document, with_keys=True, wild=wild_alter)
        for k, v in findings.items():
            trans_val = _call_callback(
                v, callback_function, function_parameters, conversion_function
            )
            # use the transformed value and apply the update to the key
            # (dont treat the lists as elements here)
            document = nested_update(
                document, k, trans_val, in_place=in_place, treat_as_element=False
            )

    return document