Coverage for hiperta_stream/utils/substring_substitution_utils.py: 83%

1import copy 

2import re 

3import warnings 

4 

5 

6def dict_defined_substring_substitution(d, external_variables=None, variable_identifier="@"): 

7 """Replaces @{variable} by variable's value, in all values of `d` if variable is a unique key in `d` or `external_variables`. 

8 

9 This is typically used to perform sub-string substitution in a loaded configuration. 

10 Keys can not be modified, only values. 

11 

12 Notes 

13 ----- 

14 - Substitution only happens inside variable of type string. 

15 - Only variables of type string, int or float can be substituted. 

16 - Variables which are defined more than once in the dictionary are ignored. 

17 

18 Parameters 

19 ---------- 

20 d : dict 

21 The bash variable syntax in the string values of `d` will be replaced. 

22 external_variables : dict, optional 

23 Dictionary of variables not defined in `d`, to use for replacement as well. 

24 variable_identifier : str 

25 Character marking a sub-string to replace. 

26 Examples: '@' to replace @{v} by v's value. 

27 

28 Returns 

29 ------- 

30 dict 

31 The modified dictionary with bash variable syntax replaced. 

32 

33 Examples 

34 -------- 

35 >>> d = { 

36 >>> "a": "b", 

37 >>> "c": {"d": "@{a}"}, 

38 >>> "e": "@{a}", 

39 >>> "f": "@{t}", 

40 >>> } 

41 >>> dict_defined_substring_substitution(d, {"t": "tt"}) 

42 {"a": "b", "c": {"d": "b"}, "e": "b", "f": "tt"} 

43 """ 

44 

45 # helper function to do the substitution 

46 # This is an adaptation of os.path.expandvars, but replaces values of keys present in `variables` 

47 # and instead of replacing environment variables. 

48 # Also doesn't allow @qdf to be replaced by qdf's value. {} are mandatory. 

49 # https://github.com/python/cpython/blob/3.11/Lib/posixpath.py 

50 if len(variable_identifier) > 1 or not variable_identifier: 50 ↛ 51line 50 didn't jump to line 51, because the condition on line 50 was never true

51 raise ValueError(f"variable identifier {variable_identifier} is not valid. It must be a single character.") 

52 

53 search = re.compile(rf"\{variable_identifier}" r"(\{[^}]*\})", re.ASCII).search 

54 

55 def substitute_in_str(string, variables): 

56 i = 0 

57 while True: 

58 m = search(string, i) 

59 if not m: 

60 break 

61 i, j = m.span(0) 

62 name = m.group(1) 

63 warning_name = name 

64 name = name[1:-1] # remove starting {, ending } 

65 try: 

66 value = variables[name] 

67 except KeyError: 

68 warnings.warn( 

69 "Value {}{} did not match any defined variables, ignoring...".format( 

70 variable_identifier, warning_name 

71 ) 

72 ) 

73 i = j 

74 else: 

75 tail = string[j:] 

76 string = string[:i] + str(value) 

77 i = len(string) 

78 string += tail 

79 return string 

80 

81 # Go through the dictionary to get all the variables that are defined only once. 

82 def get_all_unique_keys_as_if_global(d, variables, variables_already_defined): 

83 for key in d.keys(): 

84 if isinstance(d[key], dict): 

85 get_all_unique_keys_as_if_global(d[key], variables, variables_already_defined) 

86 elif isinstance(d[key], (str, int, float)): 86 ↛ 83line 86 didn't jump to line 83, because the condition on line 86 was never false

87 # variable is defined more than 1 time, remove it from vars list 

88 if key in variables_already_defined: 88 ↛ 89line 88 didn't jump to line 89, because the condition on line 88 was never true

89 if key in variables.keys(): 

90 variables.pop(key) 

91 else: 

92 variables[key] = d[key] 

93 variables_already_defined.add(key) 

94 

95 # Go through the dict to perform the substitution. 

96 def recursive_dict_vars_substitution(d, variables): 

97 for key in d.keys(): 

98 if isinstance(d[key], dict): 

99 recursive_dict_vars_substitution(d[key], variables) 

100 elif isinstance(d[key], str): 100 ↛ 97line 100 didn't jump to line 97, because the condition on line 100 was never false

101 new_value = substitute_in_str(d[key], variables) 

102 # perform substitution until the string remains unchanged, 

103 # in case several level of substitution needs doing. 

104 while new_value != d[key]: 

105 d[key] = new_value 

106 new_value = substitute_in_str(d[key], variables) 

107 

108 variables = {} if external_variables is None else copy.deepcopy(external_variables) 

109 

110 get_all_unique_keys_as_if_global(d, variables, set()) 

111 recursive_dict_vars_substitution(d, variables) 

112 

113 return d