Coverage for src/rok4_tools/joincache.py: 48%

1#!/usr/bin/env python3 

2 

3import argparse 

4import json 

5import logging 

6import os 

7import sys 

8from json.decoder import JSONDecodeError 

9 

10import jsonschema.validators 

11from jsonschema import ValidationError, validate 

12from rok4.storage import get_data_str 

13 

14from rok4_tools import __version__ 

15from rok4_tools.joincache_utils.agent import work as agent_work 

16from rok4_tools.joincache_utils.finisher import work as finisher_work 

17from rok4_tools.joincache_utils.master import work as master_work 

18 

19# Default logger 

20logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.WARNING) 

21 

22config = {} 

23args = None 

24 

25 

26def parse() -> None: 

27 """Parse call arguments and check values 

28 

29 Exit program if an error occured 

30 """ 

31 

32 global args 

33 

34 # CLI call parser 

35 parser = argparse.ArgumentParser( 

36 prog="joincache", 

37 description="Tool to generate a pyramid from other compatible pyramid", 

38 epilog="", 

39 ) 

40 

41 parser.add_argument("--version", action="version", version="%(prog)s " + __version__) 

42 

43 parser.add_argument( 

44 "--role", 

45 choices=["master", "agent", "finisher", "example", "check"], 

46 action="store", 

47 dest="role", 

48 help="Script's role", 

49 required=True, 

50 ) 

51 

52 parser.add_argument( 

53 "--conf", 

54 metavar="storage://path/to/conf.json", 

55 action="store", 

56 dest="configuration", 

57 help="Configuration file or object, JSON format", 

58 required=False, 

59 ) 

60 

61 parser.add_argument( 

62 "--split", 

63 type=int, 

64 metavar="N", 

65 action="store", 

66 dest="split", 

67 help="Split number, only required for the agent role", 

68 required=False, 

69 ) 

70 

71 args = parser.parse_args() 

72 

73 if args.role != "example" and (args.configuration is None): 

74 print("joincache: error: argument --conf is required for all roles except 'example'") 

75 sys.exit(1) 

76 

77 if args.role == "agent" and (args.split is None or args.split < 1): 

78 print( 

79 "joincache: error: argument --split is required for the agent role and have to be a positive integer" 

80 ) 

81 sys.exit(1) 

82 

83 

84def configuration() -> None: 

85 """Load configuration file 

86 

87 Raises: 

88 JSONDecodeError: Configuration is not a valid JSON file 

89 ValidationError: Configuration is not a valid JOINCACHE configuration file 

90 MissingEnvironmentError: Missing object storage informations 

91 StorageError: Storage read issue 

92 FileNotFoundError: File or object does not exist 

93 """ 

94 

95 global config 

96 

97 # Chargement du schéma JSON 

98 f = open(os.path.join(os.path.dirname(__file__), "joincache_utils", "schema.json")) 

99 schema = json.load(f) 

100 f.close() 

101 

102 # Chargement et validation de la configuration JSON 

103 config = json.loads(get_data_str(args.configuration)) 

104 validate(config, schema) 

105 

106 # Valeurs par défaut et cohérence avec l'appel 

107 if "parallelization" not in config["process"]: 

108 config["process"]["parallelization"] = 1 

109 

110 if args.role == "agent" and args.split > config["process"]["parallelization"]: 

111 raise Exception( 

112 f"Split number have to be consistent with the parallelization level: {args.split} > {config['process']['parallelization']}" 

113 ) 

114 

115 if "only_links" not in config["process"]: 

116 config["process"]["only_links"] = False 

117 

118 if "mask" not in config["process"]: 

119 config["process"]["mask"] = False 

120 config["pyramid"]["mask"] = False 

121 else: 

122 if "mask" not in config["pyramid"]: 

123 config["pyramid"]["mask"] = False 

124 elif config["process"]["mask"] == False and config["pyramid"]["mask"] == True: 

125 raise Exception( 

126 f"The new pyramid cannot have mask if masks are not used during the process" 

127 ) 

128 

129 # Logger 

130 if "logger" in config: 

131 # On supprime l'ancien logger (celui configuré par défaut) et on le reconfigure avec les nouveaux paramètres 

132 for handler in logging.root.handlers[:]: 

133 logging.root.removeHandler(handler) 

134 

135 if "file" in config["logger"]: 

136 logging.basicConfig( 

137 level=logging.getLevelName(config["logger"].get("level", "WARNING")), 

138 format=config["logger"].get("layout", "%(asctime)s %(levelname)s: %(message)s"), 

139 filename=config["logger"]["file"], 

140 ) 

141 else: 

142 logging.basicConfig( 

143 level=logging.getLevelName(config["logger"].get("level", "WARNING")), 

144 format=config["logger"].get("layout", "%(asctime)s %(levelname)s: %(message)s"), 

145 ) 

146 

147 

148def main(): 

149 """Main function 

150 

151 Return 0 if success, 1 if an error occured 

152 """ 

153 

154 parse() 

155 

156 if args.role == "example": 

157 # On veut juste afficher la configuration en exemple 

158 f = open(os.path.join(os.path.dirname(__file__), "joincache_utils/example.json")) 

159 print(f.read()) 

160 f.close 

161 sys.exit(0) 

162 

163 # Configuration 

164 try: 

165 configuration() 

166 

167 except JSONDecodeError as e: 

168 logging.error(f"{args.configuration} is not a valid JSON file: {e}") 

169 sys.exit(1) 

170 

171 except ValidationError as e: 

172 logging.error(f"{args.configuration} is not a valid configuration file: {e}") 

173 sys.exit(1) 

174 

175 except Exception as e: 

176 logging.error(e) 

177 sys.exit(1) 

178 

179 if args.role == "check": 

180 # On voulait juste valider le fichier de configuration, c'est chose faite 

181 # Si on est là c'est que tout est bon 

182 print("Valid configuration !") 

183 sys.exit(0) 

184 

185 # Work 

186 try: 

187 if args.role == "master": 

188 master_work(config) 

189 elif args.role == "agent": 

190 agent_work(config, args.split) 

191 elif args.role == "finisher": 

192 finisher_work(config) 

193 

194 except Exception as e: 

195 logging.error(e) 

196 sys.exit(1) 

197 

198 sys.exit(0)