1 | from jinja2 import Environment |
---|
2 | from jinja2.loaders import FileSystemLoader |
---|
3 | from jinja2 import Template |
---|
4 | |
---|
5 | import tempfile |
---|
6 | |
---|
7 | import os |
---|
8 | import threading |
---|
9 | import datetime |
---|
10 | import pexpect |
---|
11 | import re |
---|
12 | import json |
---|
13 | |
---|
14 | import BaseHTTPServer |
---|
15 | from SimpleHTTPServer import SimpleHTTPRequestHandler |
---|
16 | from multiprocessing import Process |
---|
17 | import socket |
---|
18 | from urllib2 import urlopen |
---|
19 | import string |
---|
20 | import subprocess |
---|
21 | |
---|
22 | class MirrorManager: |
---|
23 | |
---|
24 | |
---|
25 | def __init__(self): |
---|
26 | #Default values |
---|
27 | self.defaultpath = '/etc/lliurex-mirror/' |
---|
28 | self.debmirrorconfpath = os.path.join(self.defaultpath,'debmirror') |
---|
29 | self.configpath = os.path.join(self.defaultpath,'conf') |
---|
30 | self.distro="llx16" |
---|
31 | self.httpd = {} |
---|
32 | self.debmirrorprocess = None |
---|
33 | |
---|
34 | self.tpl_env = Environment(loader=FileSystemLoader('/usr/share/n4d/templates/lliurex-mirror')) |
---|
35 | self.update_thread=threading.Thread() |
---|
36 | self.get_mirror_thread = threading.Thread() |
---|
37 | self.percentage=(0,None) |
---|
38 | self.exportpercentage = 0 |
---|
39 | self.mirrorworking = None |
---|
40 | self.webserverprocess = {} |
---|
41 | self.defaultmirrorinfo = {"status_mirror":"New","last_mirror_date":None,"mirror_size":0,"progress":0} |
---|
42 | self.valid_chars = "-_.%s%s" % (string.ascii_letters, string.digits) |
---|
43 | self.default_mirror_config = ''' |
---|
44 | { |
---|
45 | "NAME": "", |
---|
46 | "BANNER": "", |
---|
47 | "ORIGS" : {"1":"lliruex.net/xenial","2":"","3":""}, # 1 ORIGINAL ; 2 LOCALFILESYSTEM; 3 REMOTEURL |
---|
48 | "ARCHITECTURES": [ "amd64", "i386"], |
---|
49 | "SECTIONS": ["main", "main/debian-installer", "universe", "restricted", "multiverse", "partner"], |
---|
50 | "MIRROR_PATH": "/net/mirror/llx16", |
---|
51 | "DISTROS": ["xenial","xenial-updates","xenial-security"], |
---|
52 | "IGN_GPG":1, |
---|
53 | "IGN_RELEASE":0, |
---|
54 | "CHK_MD5":0, |
---|
55 | "CURRENT_UPDATE_OPTION":"1" |
---|
56 | }''' |
---|
57 | |
---|
58 | #def init |
---|
59 | |
---|
60 | def startup(self,options): |
---|
61 | self.n4d_vars=objects["VariablesManager"] |
---|
62 | self.variable=objects["VariablesManager"].get_variable("LLIUREXMIRROR") |
---|
63 | |
---|
64 | |
---|
65 | if self.variable==None: |
---|
66 | try: |
---|
67 | self.n4d_vars.add_variable("LLIUREXMIRROR",{},"","Lliurex Mirror info variable","n4d-lliurex-mirror") |
---|
68 | except Exception as e: |
---|
69 | pass |
---|
70 | |
---|
71 | if type(self.variable)!=type({}): |
---|
72 | self.variable={} |
---|
73 | |
---|
74 | try: |
---|
75 | for repo in self.get_available_mirrors()['msg']: |
---|
76 | if self.variable.has_key(repo) and self.variable[repo].has_key("status_mirror") and self.variable[repo]["status_mirror"] == "Working": |
---|
77 | if not self.update_thread.isAlive(): |
---|
78 | self.variable[repo]["status_mirror"] = "Error" |
---|
79 | self.n4d_vars.set_variable("LLIUREXMIRROR",self.variable) |
---|
80 | else: |
---|
81 | if not self.variable.has_key(repo): |
---|
82 | self.variable[repo] = self.defaultmirrorinfo |
---|
83 | except Exception as e: |
---|
84 | pass |
---|
85 | #def startup |
---|
86 | |
---|
87 | def apt(self): |
---|
88 | # executed after apt operations |
---|
89 | pass |
---|
90 | |
---|
91 | #def apt |
---|
92 | |
---|
93 | # service test and backup functions # |
---|
94 | |
---|
95 | def test(self): |
---|
96 | |
---|
97 | pass |
---|
98 | |
---|
99 | #def test |
---|
100 | |
---|
101 | def backup(self): |
---|
102 | |
---|
103 | pass |
---|
104 | |
---|
105 | #def backup |
---|
106 | |
---|
107 | def restore(self): |
---|
108 | pass |
---|
109 | #def restore |
---|
110 | |
---|
111 | def set_cname(self): |
---|
112 | #Get template |
---|
113 | template = self.tpl_env.get_template("cname") |
---|
114 | list_variables = {} |
---|
115 | |
---|
116 | list_variables = self.n4d_vars.get_variable_list(['INTERNAL_DOMAIN','HOSTNAME']) |
---|
117 | for x in list_variables.keys(): |
---|
118 | if list_variables[x] == None: |
---|
119 | return {'status':False,'msg':'Variable ' + x + ' not defined'} |
---|
120 | |
---|
121 | #Encode vars to UTF-8 |
---|
122 | string_template = template.render(list_variables).encode('UTF-8') |
---|
123 | #Open template file |
---|
124 | fd, tmpfilepath = tempfile.mkstemp() |
---|
125 | new_export_file = open(tmpfilepath,'w') |
---|
126 | new_export_file.write(string_template) |
---|
127 | new_export_file.close() |
---|
128 | os.close(fd) |
---|
129 | #Write template values |
---|
130 | n4d_mv(tmpfilepath,'/var/lib/dnsmasq/config/cname-mirror',True,'root','root','0644',False ) |
---|
131 | |
---|
132 | return {'status':True,'msg':'Set mirror cname'} |
---|
133 | #def set_cname |
---|
134 | |
---|
135 | def update(self,ip,distro=None,callback_args=None): |
---|
136 | |
---|
137 | if distro==None: |
---|
138 | distro=self.distro |
---|
139 | |
---|
140 | if self.update_thread.is_alive(): |
---|
141 | return {'status':False,'msg':'Lliurex-mirror (n4d instance) is running'} |
---|
142 | |
---|
143 | self.percentage=(0,None) |
---|
144 | self.update_thread=threading.Thread(target=self._update,args=(ip,distro,callback_args,)) |
---|
145 | self.update_thread.daemon=True |
---|
146 | self.update_thread.start() |
---|
147 | |
---|
148 | return {'status':True,'msg':'running'} |
---|
149 | |
---|
150 | #def update |
---|
151 | |
---|
152 | def _update(self,ip,distro,callback_args): |
---|
153 | if not self.variable.has_key(distro): |
---|
154 | self.variable[distro]=self.defaultmirrorinfo |
---|
155 | # link config debmirror to correct path with distro name |
---|
156 | self.variable[distro]['status_mirror'] = "Working" |
---|
157 | self.variable[distro]['progress'] = 0 |
---|
158 | self.variable[distro]['exception_msg'] = "" |
---|
159 | self.n4d_vars.set_variable("LLIUREXMIRROR",self.variable) |
---|
160 | self.build_debmirror_config(distro) |
---|
161 | if os.path.lexists('/etc/debmirror.conf'): |
---|
162 | os.remove('/etc/debmirror.conf') |
---|
163 | os.symlink(os.path.join(self.debmirrorconfpath,distro),'/etc/debmirror.conf') |
---|
164 | self.mirrorworking = distro |
---|
165 | fd = open('/var/log/lliurex-mirror.log','a') |
---|
166 | import datetime |
---|
167 | fd.write("MIRROR START AT " + str(datetime.datetime.now())+ " \n") |
---|
168 | fd.flush() |
---|
169 | errors_found = False |
---|
170 | self.debmirrorprocess=pexpect.spawn("/usr/bin/debmirror") |
---|
171 | download_packages = False |
---|
172 | emergency_counter = 0 |
---|
173 | try: |
---|
174 | objects["ZCenterVariables"].add_pulsating_color("lliurexmirror") |
---|
175 | except: |
---|
176 | pass |
---|
177 | while True: |
---|
178 | try: |
---|
179 | emergency_counter += 1 |
---|
180 | if emergency_counter > 100: |
---|
181 | download_packages = True |
---|
182 | self.debmirrorprocess.expect('\n',timeout=480) |
---|
183 | line =self.debmirrorprocess.before |
---|
184 | |
---|
185 | fd.write(line + "\n") |
---|
186 | fd.flush() |
---|
187 | if line.find("Files to download") >= 0 : |
---|
188 | download_packages = True |
---|
189 | line1=line.strip("\n") |
---|
190 | if download_packages: |
---|
191 | if line1.startswith("[") and line1[5] == "]": |
---|
192 | self.percentage=(int(line1[1:4].strip()),self.debmirrorprocess.exitstatus) |
---|
193 | self.variable[distro]['progress'] = self.percentage[0] |
---|
194 | self.n4d_vars.set_variable("LLIUREXMIRROR",self.variable) |
---|
195 | if line1.startswith("Everything OK"): |
---|
196 | self.percentage=(100,self.debmirrorprocess.exitstatus) |
---|
197 | self.variable[distro]['progress'] = 100 |
---|
198 | self.n4d_vars.set_variable("LLIUREXMIRROR",self.variable) |
---|
199 | |
---|
200 | if errors_found: |
---|
201 | e=Exception(line1) |
---|
202 | raise e |
---|
203 | if line1.startswith("Errors"): |
---|
204 | errors_found = True |
---|
205 | |
---|
206 | |
---|
207 | except pexpect.EOF: |
---|
208 | line1 = self.debmirrorprocess.before |
---|
209 | if line1 != "" and line1.startswith("[") and line1[5] == "]": |
---|
210 | self.percentage=(int(line1[1:4].strip()),self.debmirrorprocess.exitstatus) |
---|
211 | self.debmirrorprocess.close() |
---|
212 | status = self.debmirrorprocess.exitstatus |
---|
213 | self.percentage=(self.percentage[0],status) |
---|
214 | self.variable[distro]['progress'] = self.percentage[0] |
---|
215 | self.variable[distro]['status_mirror'] = "Ok" if status == 0 else "Error" |
---|
216 | self.n4d_vars.set_variable("LLIUREXMIRROR",self.variable) |
---|
217 | break |
---|
218 | except Exception as e: |
---|
219 | print e |
---|
220 | self.variable[distro]['status_mirror'] = "Error" |
---|
221 | self.variable[distro]["exception_msg"] = str(e) |
---|
222 | status = self.debmirrorprocess.exitstatus |
---|
223 | self.percentage=(self.percentage[0],str(e)) |
---|
224 | self.n4d_vars.set_variable("LLIUREXMIRROR",self.variable) |
---|
225 | break |
---|
226 | fd.close() |
---|
227 | if type(callback_args) == type({}): |
---|
228 | if callback_args.has_key('port'): |
---|
229 | import xmlrpclib as x |
---|
230 | c = x.ServerProxy('https://' + ip + ':9779') |
---|
231 | c.stop_webserver('','MirrorManager',callback_args['port']) |
---|
232 | |
---|
233 | self.download_time_file(distro) |
---|
234 | self.set_mirror_info(distro) |
---|
235 | self.mirrorworking = None |
---|
236 | try: |
---|
237 | objects["ZCenterVariables"].remove_pulsating_color("lliurexmirror") |
---|
238 | except: |
---|
239 | pass |
---|
240 | |
---|
241 | #def _update |
---|
242 | |
---|
243 | def is_alive(self): |
---|
244 | return {'status':self.update_thread.is_alive(),'msg':self.mirrorworking} |
---|
245 | #def is_alive |
---|
246 | |
---|
247 | def set_mirror_info(self,distro=None): |
---|
248 | |
---|
249 | if distro!=None: |
---|
250 | distro=distro |
---|
251 | else: |
---|
252 | distro=self.distro |
---|
253 | |
---|
254 | configpath = os.path.join(self.configpath, distro + ".json") |
---|
255 | config = json.load(open(configpath,'r')) |
---|
256 | |
---|
257 | mirrorpath = config["MIRROR_PATH"] |
---|
258 | #self.n4d_vars.set_variable("ZEROCENTERINTERNAL",self.internal_variable) |
---|
259 | |
---|
260 | MIRROR_DATE=datetime.date.today().strftime("%d/%m/%Y") |
---|
261 | MIRROR_SIZE=self.get_size(mirrorpath) |
---|
262 | |
---|
263 | self.variable[distro]["last_mirror_date"]=MIRROR_DATE |
---|
264 | self.variable[distro]["mirror_size"]=str(MIRROR_SIZE) |
---|
265 | self.variable[distro]["progress"]=self.percentage[0] |
---|
266 | |
---|
267 | self.n4d_vars.set_variable("LLIUREXMIRROR",self.variable) |
---|
268 | |
---|
269 | #set_custom_text(self,app,text): |
---|
270 | txt="Updated on: " + str(MIRROR_DATE) |
---|
271 | txt+=" # Size: %.2fGB"%MIRROR_SIZE |
---|
272 | try: |
---|
273 | objects["ZCenterVariables"].set_custom_text("lliurexmirror",txt) |
---|
274 | abstract=open('/var/log/lliurex/lliurex-mirror.log','w') |
---|
275 | abstract.write(txt+"\n") |
---|
276 | abstract.close() |
---|
277 | except Exception as e: |
---|
278 | pass |
---|
279 | |
---|
280 | #def set_mirror_info(self): |
---|
281 | |
---|
282 | def update_size_info(self,distro): |
---|
283 | if distro!=None: |
---|
284 | distro=distro |
---|
285 | else: |
---|
286 | distro=self.distro |
---|
287 | |
---|
288 | configpath = os.path.join(self.configpath, distro + ".json") |
---|
289 | config = json.load(open(configpath,'r')) |
---|
290 | |
---|
291 | mirrorpath = config["MIRROR_PATH"] |
---|
292 | MIRROR_SIZE=self.get_size(mirrorpath) |
---|
293 | self.variable[distro]["mirror_size"]=str(MIRROR_SIZE) |
---|
294 | self.n4d_vars.set_variable("LLIUREXMIRROR",self.variable) |
---|
295 | return {'status':True,'msg':MIRROR_SIZE} |
---|
296 | |
---|
297 | |
---|
298 | def get_size(self,start_path = '.'): |
---|
299 | |
---|
300 | total_size = 0 |
---|
301 | try: |
---|
302 | for dirpath, dirnames, filenames in os.walk(start_path): |
---|
303 | for f in filenames: |
---|
304 | fp = os.path.join(dirpath, f) |
---|
305 | total_size += os.path.getsize(fp) |
---|
306 | |
---|
307 | total_size/=1024*1024*1024.0 |
---|
308 | return total_size |
---|
309 | except: |
---|
310 | return 0 |
---|
311 | |
---|
312 | #def get_size(start_path = '.'): |
---|
313 | |
---|
314 | def search_field(self,filepath,fieldname): |
---|
315 | try: |
---|
316 | f = open(filepath,'r') |
---|
317 | needle = None |
---|
318 | lines = f.readlines() |
---|
319 | for x in lines: |
---|
320 | if re.match('\s*'+fieldname,x): |
---|
321 | needle = x.strip() |
---|
322 | return needle |
---|
323 | except: |
---|
324 | return None |
---|
325 | # def search_field |
---|
326 | |
---|
327 | def get_mirror_architecture(self,distro): |
---|
328 | |
---|
329 | configpath = os.path.join(self.configpath,distro + ".json") |
---|
330 | config = json.load(open(configpath,'r')) |
---|
331 | if not os.path.lexists(configpath): |
---|
332 | return {'status':False,'msg':'not exists debmirror.conf to '+ distro } |
---|
333 | |
---|
334 | if "ARCHITECTURES" in config.keys(): |
---|
335 | return {'status':True,'msg':config["ARCHITECTURES"] } |
---|
336 | |
---|
337 | return {'status':False,'msg':"debmirror.conf hasn't architecture variable" } |
---|
338 | #def get_mirror_architecture |
---|
339 | |
---|
340 | def set_mirror_architecture(self,distro,archs): |
---|
341 | configpath = os.path.join(self.configpath,distro + ".json") |
---|
342 | |
---|
343 | config = json.load(open(configpath,'r')) |
---|
344 | |
---|
345 | config['ARCHITECTURES'] = archs |
---|
346 | |
---|
347 | |
---|
348 | f=open(configpath,"w") |
---|
349 | |
---|
350 | data=unicode(json.dumps(config,indent=4,encoding="utf-8",ensure_ascii=False)).encode("utf-8") |
---|
351 | f.write(data) |
---|
352 | f.close() |
---|
353 | |
---|
354 | self.build_debmirror_config(distro) |
---|
355 | return {'status':True,'msg':'set architecture'} |
---|
356 | |
---|
357 | #def set_mirror_architecture |
---|
358 | |
---|
359 | def get_mirror_orig(self,distro,option): |
---|
360 | |
---|
361 | configpath = os.path.join(self.configpath,distro + ".json") |
---|
362 | config = json.load(open(configpath,'r')) |
---|
363 | if not os.path.lexists(configpath): |
---|
364 | return {'status':False,'msg':'not exists debmirror.conf to '+ distro } |
---|
365 | |
---|
366 | if "ORIGS" in config.keys(): |
---|
367 | return {'status':True,'msg':config["ORIGS"][option] } |
---|
368 | |
---|
369 | return {'status':False,'msg':"debmirror.conf hasn't orig variable" } |
---|
370 | #def get_mirror_from |
---|
371 | |
---|
372 | def set_mirror_orig(self,distro,url,option): |
---|
373 | if url == None: |
---|
374 | return {'status':False,'msg':'url is None'} |
---|
375 | configpath = os.path.join(self.configpath, distro + ".json") |
---|
376 | config = json.load(open(configpath,'r')) |
---|
377 | config['ORIGS'][option] = url |
---|
378 | |
---|
379 | f=open(configpath,"w") |
---|
380 | data=unicode(json.dumps(config,indent=4,encoding="utf-8",ensure_ascii=False)).encode("utf-8") |
---|
381 | f.write(data) |
---|
382 | f.close() |
---|
383 | |
---|
384 | self.build_debmirror_config(distro) |
---|
385 | return {'status':True,'msg':'set orig'} |
---|
386 | #def set_mirror_architecture |
---|
387 | |
---|
388 | def get_option_update(self,distro): |
---|
389 | configpath = os.path.join(self.configpath,distro + ".json") |
---|
390 | config = json.load(open(configpath,'r')) |
---|
391 | if not os.path.lexists(configpath): |
---|
392 | return {'status':False,'msg':'not exists debmirror.conf to '+ distro } |
---|
393 | |
---|
394 | if "CURRENT_UPDATE_OPTION" in config.keys(): |
---|
395 | return {'status':True,'msg':config["CURRENT_UPDATE_OPTION"] } |
---|
396 | |
---|
397 | return {'status':False,'msg':"debmirror.conf hasn't option update variable" } |
---|
398 | #def get_option_update |
---|
399 | |
---|
400 | def set_option_update(self,distro,option): |
---|
401 | configpath = os.path.join(self.configpath, distro + ".json") |
---|
402 | config = json.load(open(configpath,'r')) |
---|
403 | #Sanitize mirror url if it's a custom one |
---|
404 | customMirror=config['ORIGS']['3'] |
---|
405 | if "http" in customMirror: |
---|
406 | customMirror=customMirror.split('//')[-1] |
---|
407 | config['ORIGS']['3']=customMirror |
---|
408 | config['CURRENT_UPDATE_OPTION'] = str(option) |
---|
409 | |
---|
410 | f=open(configpath,"w") |
---|
411 | data=unicode(json.dumps(config,indent=4,encoding="utf-8",ensure_ascii=False)).encode("utf-8") |
---|
412 | f.write(data) |
---|
413 | f.close() |
---|
414 | |
---|
415 | self.build_debmirror_config(distro) |
---|
416 | return {'status':True,'msg':'set update option'} |
---|
417 | #def set_option_update |
---|
418 | |
---|
419 | def get_percentage(self,distro): |
---|
420 | if self.variable.has_key(distro): |
---|
421 | return {'status':True,'msg':self.variable[distro]['progress']} |
---|
422 | else: |
---|
423 | return {'status':False,'msg':'this repo nos has been configured'} |
---|
424 | #def get_percentage |
---|
425 | |
---|
426 | |
---|
427 | def build_debmirror_config(self,distro): |
---|
428 | result = self.render_debmirror_config(distro) |
---|
429 | string_template = result['msg'] |
---|
430 | f = open(os.path.join(self.debmirrorconfpath,distro),'w') |
---|
431 | f.write(string_template) |
---|
432 | f.close() |
---|
433 | #def build_debmirror_config |
---|
434 | |
---|
435 | def render_debmirror_config(self,arg): |
---|
436 | if type(arg) == type(""): |
---|
437 | return self._render_debmirror_config_distro(arg) |
---|
438 | if type(arg) == type({}): |
---|
439 | return self._render_debmirror_config_values(arg) |
---|
440 | #def render_debmirror_config |
---|
441 | |
---|
442 | def _render_debmirror_config_distro(self,distro): |
---|
443 | template = self.tpl_env.get_template('debmirror.conf') |
---|
444 | configpath = os.path.join(self.configpath,distro + ".json") |
---|
445 | config = json.load(open(configpath,'r')) |
---|
446 | return {'status':True,'msg':template.render(config).encode('utf-8')} |
---|
447 | #def render_debmirror_config |
---|
448 | |
---|
449 | def _render_debmirror_config_values(self,config): |
---|
450 | template = self.tpl_env.get_template('debmirror.conf') |
---|
451 | return {'status':True,'msg':template.render(config).encode('utf-8')} |
---|
452 | #def _render_debmirror_config_values |
---|
453 | |
---|
454 | def enable_webserver_into_folder(self,path): |
---|
455 | |
---|
456 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
---|
457 | s.bind(('localhost', 0)) |
---|
458 | addr, port = s.getsockname() |
---|
459 | s.close() |
---|
460 | self.webserverprocess[str(port)] = Process(target=self._enable_webserver_into_folder,args=(port,path,)) |
---|
461 | self.webserverprocess[str(port)].start() |
---|
462 | return {'status':True,'msg':port} |
---|
463 | #enable_webserver_into_folder |
---|
464 | |
---|
465 | def _enable_webserver_into_folder(self,port,path): |
---|
466 | try: |
---|
467 | iface = '127.0.0.1' |
---|
468 | sock = (iface,port) |
---|
469 | proto = "HTTP/1.0" |
---|
470 | os.chdir(path) |
---|
471 | handler = SimpleHTTPRequestHandler |
---|
472 | handler.protocol_version = proto |
---|
473 | self.httpd[str(port)] = BaseHTTPServer.HTTPServer(sock,handler) |
---|
474 | self.httpd[str(port)].serve_forever() |
---|
475 | except Exception, e: |
---|
476 | return None |
---|
477 | #_enable_webserver_into_folder |
---|
478 | |
---|
479 | def stop_webserver(self,port): |
---|
480 | if self.webserverprocess.has_key(port): |
---|
481 | self.webserverprocess[port].terminate() |
---|
482 | self.webserverprocess.pop(port) |
---|
483 | return {'status':True,'msg':'Server stopped'} |
---|
484 | return {'status':False,'msg':'Server not exists'} |
---|
485 | #stop_webserver |
---|
486 | |
---|
487 | def set_checksum_validation(self,distro,status): |
---|
488 | configpath = os.path.join(self.configpath, distro + ".json") |
---|
489 | config = json.load(open(configpath,'r')) |
---|
490 | config['CHK_MD5'] = status |
---|
491 | |
---|
492 | f=open(configpath,"w") |
---|
493 | data=unicode(json.dumps(config,indent=4,encoding="utf-8",ensure_ascii=False)).encode("utf-8") |
---|
494 | f.write(data) |
---|
495 | f.close() |
---|
496 | |
---|
497 | self.build_debmirror_config(distro) |
---|
498 | return {'status':True,'msg':'set checksum validation'} |
---|
499 | #set_checksum_validation |
---|
500 | |
---|
501 | def get_checksum_validation(self,distro): |
---|
502 | |
---|
503 | configpath = os.path.join(self.configpath,distro + ".json") |
---|
504 | config = json.load(open(configpath,'r')) |
---|
505 | if not os.path.lexists(configpath): |
---|
506 | return {'status':False,'msg':'not exists debmirror.conf to '+ distro } |
---|
507 | if "IGN_GPG" in config.keys(): |
---|
508 | return {'status':True,'msg':config["CHK_MD5"] } |
---|
509 | |
---|
510 | return {'status':False,'msg':"debmirror.conf hasn't orig variable" } |
---|
511 | #get_checksum_validation |
---|
512 | |
---|
513 | def get_available_mirrors(self): |
---|
514 | versions = os.listdir(self.configpath) |
---|
515 | versions = [ version.replace('.json','') for version in versions if version.endswith('.json')] |
---|
516 | return {'status':True,'msg':versions} |
---|
517 | |
---|
518 | def stopupdate(self): |
---|
519 | try: |
---|
520 | self.debmirrorprocess.terminate() |
---|
521 | return {'status':True,'msg':'debmirror stopped'} |
---|
522 | except Exception as e: |
---|
523 | return {'status':False,'msg':str(e)} |
---|
524 | |
---|
525 | def stopgetmirror(self): |
---|
526 | try: |
---|
527 | self.get_mirror_process.terminate() |
---|
528 | return {'status':True,'msg':'debmirror stopped'} |
---|
529 | except Exception as e: |
---|
530 | return {'status':False,'msg':str(e)} |
---|
531 | |
---|
532 | def download_time_file(self,distro): |
---|
533 | |
---|
534 | configpath = os.path.join(self.configpath,distro + ".json") |
---|
535 | config = json.load(open(configpath,'r')) |
---|
536 | path=config["MIRROR_PATH"] |
---|
537 | f="time-of-last-update" |
---|
538 | dest=os.path.join(path,f) |
---|
539 | |
---|
540 | orig_mirror=self.get_mirror_orig(distro,"1") |
---|
541 | url_mirror="http://"+os.path.join(orig_mirror['msg'],f) |
---|
542 | |
---|
543 | return self.get_time_file(url_mirror,dest) |
---|
544 | |
---|
545 | # # def download_time_file |
---|
546 | |
---|
547 | |
---|
548 | def get_time_file(self,url,dest): |
---|
549 | |
---|
550 | try: |
---|
551 | r=urlopen(url) |
---|
552 | f=open(dest,"wb") |
---|
553 | f.write(r.read()) |
---|
554 | f.close() |
---|
555 | r.close() |
---|
556 | return {'status':True,'msg':dest + 'successfully downloaded.'} |
---|
557 | |
---|
558 | except Exception as e: |
---|
559 | return {'status':False,'msg':'Error downloading' + dest + ':' + str(e)} |
---|
560 | |
---|
561 | # def get_time_file |
---|
562 | |
---|
563 | def is_update_available(self,distro): |
---|
564 | |
---|
565 | configpath = os.path.join(self.configpath,distro + ".json") |
---|
566 | config = json.load(open(configpath,'r')) |
---|
567 | path = config["MIRROR_PATH"] |
---|
568 | file_time_name = "time-of-last-update" |
---|
569 | file_local_mirror = os.path.join(path,file_time_name) |
---|
570 | |
---|
571 | |
---|
572 | if os.path.isfile(file_local_mirror): |
---|
573 | url_pool = "http://"+os.path.join(config["ORIGS"]['1'],file_time_name) |
---|
574 | file_pool = os.path.join("/tmp",file_time_name) |
---|
575 | |
---|
576 | exist_file_pool = self.get_time_file(url_pool,file_pool) |
---|
577 | if exist_file_pool['status']: |
---|
578 | file_local_mirror_content=open(file_local_mirror,"r") |
---|
579 | file_local_miror_datetime=(file_local_mirror_content.readline().strip()).split("_") |
---|
580 | file_pool_content=open(file_pool,'r') |
---|
581 | file_pool_datetime=(file_pool_content.readline().strip()).split("_") |
---|
582 | file_local_mirror_content.close() |
---|
583 | file_pool_content.close() |
---|
584 | |
---|
585 | date_local_mirror=datetime.datetime.strptime(file_local_miror_datetime[0],"%Y/%m/%d") |
---|
586 | date_pool=datetime.datetime.strptime(file_pool_datetime[0],"%Y/%m/%d") |
---|
587 | |
---|
588 | if date_local_mirror==date_pool: |
---|
589 | time_local_mirror=datetime.datetime.strptime(file_local_miror_datetime[1],"%H:%M") |
---|
590 | time_pool=datetime.datetime.strptime(file_pool_datetime[1],"%H:%M") |
---|
591 | |
---|
592 | if time_local_mirror<time_pool: |
---|
593 | return {'status':False,'msg':'Mirror not updated','action':'update'} |
---|
594 | else: |
---|
595 | return {'status':True,'msg':'Mirror is updated','action':'nothing'} |
---|
596 | |
---|
597 | elif date_local_mirror<date_pool: |
---|
598 | return {'status':False,'msg':'Mirror not updated','action':'update'} |
---|
599 | else: |
---|
600 | return {'status':True,'msg':'Mirror is updated','action':'nothing'} |
---|
601 | else: |
---|
602 | return {'status':False,'msg':exist_file_pool['msg'],'action':'nothing'} |
---|
603 | |
---|
604 | else: |
---|
605 | return {'status':False,'msg':file_local_mirror + ' does not exist.','action':'nothing'} |
---|
606 | |
---|
607 | # def is_update_available |
---|
608 | |
---|
609 | def new_mirror_config(self,config): |
---|
610 | name = config["NAME"].lower().strip() |
---|
611 | name = ''.join(c for c in name if c in self.valid_chars) |
---|
612 | |
---|
613 | # Checks |
---|
614 | if name == "": |
---|
615 | return {'status':False,'msg':"Name can't void"} |
---|
616 | while True: |
---|
617 | newconfigpath = os.path.join(self.configpath,name + '.json') |
---|
618 | if not os.path.lexists(newconfigpath): |
---|
619 | break |
---|
620 | name = name + "1" |
---|
621 | |
---|
622 | data=unicode(json.dumps(config,indent=4,encoding="utf-8",ensure_ascii=False)).encode("utf-8") |
---|
623 | f = open(newconfigpath,'w') |
---|
624 | f.write(data) |
---|
625 | f.close() |
---|
626 | self.variable[name] = self.defaultmirrorinfo |
---|
627 | return {'status':True,'msg':name} |
---|
628 | #def new_mirror_config |
---|
629 | |
---|
630 | def get_all_configs(self): |
---|
631 | versions = os.listdir(self.configpath) |
---|
632 | allconfigs = {} |
---|
633 | for version in versions: |
---|
634 | configfile = os.path.join(self.configpath,version) |
---|
635 | f = open(configfile,'r') |
---|
636 | allconfigs[version.replace('.json','')] = json.load(f) |
---|
637 | f.close() |
---|
638 | #Sanitize mirror url if it's a custom one |
---|
639 | customMirror=allconfigs[version.replace('.json','')]['ORIGS']['3'] |
---|
640 | if "http" in customMirror: |
---|
641 | customMirror=customMirror.split('//')[-1] |
---|
642 | allconfigs[version.replace('.json','')]['ORIGS']['3']=customMirror |
---|
643 | return {'status':True,'msg':allconfigs} |
---|
644 | #def get_all_configs |
---|
645 | |
---|
646 | def update_mirror_config(self,mirror,config): |
---|
647 | configpath = os.path.join(self.configpath,mirror + ".json") |
---|
648 | |
---|
649 | f=open(configpath,"w") |
---|
650 | |
---|
651 | data=unicode(json.dumps(config,indent=4,encoding="utf-8",ensure_ascii=False)).encode("utf-8") |
---|
652 | f.write(data) |
---|
653 | f.close() |
---|
654 | |
---|
655 | return {'status':True,'msg':'Updated config'} |
---|
656 | #def update_mirror_config |
---|
657 | |
---|
658 | def get_client_ip(self,ip): |
---|
659 | return {'status':True,'msg':ip} |
---|
660 | #def get_client_ip |
---|
661 | |
---|
662 | def is_alive_get_mirror(self): |
---|
663 | return {'status':self.get_mirror_thread.is_alive(),'msg':self.exportpercentage} |
---|
664 | #def is_alive_get_mirror |
---|
665 | |
---|
666 | def get_mirror(self,config_path,callback_args): |
---|
667 | self.get_mirror_thread = threading.Thread(target=self._get_mirror,args=(config_path,callback_args,)) |
---|
668 | self.get_mirror_thread.daemon = True |
---|
669 | self.get_mirror_thread.start() |
---|
670 | #def get_mirror |
---|
671 | |
---|
672 | def _get_mirror(self,config_path,callback_args): |
---|
673 | self.get_mirror_process = pexpect.spawn("/usr/bin/debmirror --config-file="+config_path) |
---|
674 | while True: |
---|
675 | try: |
---|
676 | self.get_mirror_process.expect('\n') |
---|
677 | line =self.get_mirror_process.before |
---|
678 | line1=line.strip("\n") |
---|
679 | if line1.startswith("[") and line1[5] == "]": |
---|
680 | self.exportpercentage = (int(line1[1:4].strip()),self.get_mirror_process.exitstatus) |
---|
681 | except pexpect.EOF: |
---|
682 | line1 = self.get_mirror_process.before |
---|
683 | if line1 != "" and line1.startswith("[") and line1[5] == "]": |
---|
684 | self.exportpercentage=(int(line1[1:4].strip()),self.get_mirror_process.exitstatus) |
---|
685 | self.get_mirror_process.close() |
---|
686 | status = self.get_mirror_process.exitstatus |
---|
687 | self.exportpercentage=(self.exportpercentage[0],status) |
---|
688 | break |
---|
689 | except Exception as e: |
---|
690 | break |
---|
691 | if callback_args.has_key('port') and callback_args.has_key('ip'): |
---|
692 | import xmlrpclib as x |
---|
693 | c = x.ServerProxy('https://' + callback_args['ip'] + ':9779') |
---|
694 | c.stop_webserver('','MirrorManager',callback_args['port']) |
---|
695 | #def _get |
---|
696 | |
---|
697 | def get_last_log(self): |
---|
698 | import base64 |
---|
699 | p = subprocess.Popen('lliurex-mirror-get-last-log',stdout=subprocess.PIPE) |
---|
700 | path = p.communicate()[0].strip() |
---|
701 | f = open(path,'r') |
---|
702 | content = f.readlines() |
---|
703 | onelinecontent = ''.join(content) |
---|
704 | return {'status':True,'msg':base64.b64encode(onelinecontent)} |
---|
705 | |
---|
706 | |
---|
707 | |
---|