1 | import subprocess |
---|
2 | import cmd |
---|
3 | import configparser |
---|
4 | import os |
---|
5 | import shutil |
---|
6 | import urllib |
---|
7 | import urllib.request |
---|
8 | from bs4 import BeautifulSoup |
---|
9 | import sys |
---|
10 | import mmap |
---|
11 | import re |
---|
12 | import tempfile |
---|
13 | import fileinput |
---|
14 | |
---|
15 | class MetaMaker(cmd.Cmd): |
---|
16 | intro = "Welcome to Lliurex Meta Maker \n" |
---|
17 | prompt = 'LlxMeta: ' |
---|
18 | flavours = [] |
---|
19 | root = os.getcwd() |
---|
20 | config = None |
---|
21 | seeds = {} |
---|
22 | structure = {} |
---|
23 | |
---|
24 | def completenames(self,text,*ignores): |
---|
25 | lst = cmd.Cmd.completenames(self,text,*ignores) |
---|
26 | return [a for a in lst if a != 'EOF'] |
---|
27 | |
---|
28 | def loadStructure(self): |
---|
29 | ''' |
---|
30 | load on self.structure seeds and depends |
---|
31 | self.structure = { |
---|
32 | "llx-base":[], |
---|
33 | "llx-common":[], |
---|
34 | "llx-desktop":["llx-base","llx-common"] |
---|
35 | } |
---|
36 | ''' |
---|
37 | structurefiles = [] |
---|
38 | for (dirpath,dirname,filelist) in os.walk(self.root+"/seeds"): |
---|
39 | for seed in filelist: |
---|
40 | if seed.lower() == "structure": |
---|
41 | structurefiles.append(dirpath + "/" + seed) |
---|
42 | |
---|
43 | for strucfilepath in structurefiles: |
---|
44 | fd = open(strucfilepath,'r') |
---|
45 | content = list(map(str.rstrip,fd.readlines())) |
---|
46 | for line in content: |
---|
47 | seed = line.split(":") |
---|
48 | if len(seed) > 1: |
---|
49 | depends = seed[1].lstrip().split(" ") |
---|
50 | self.structure[seed[0]] = depends if depends != [""] else [] |
---|
51 | |
---|
52 | def loadConfig(self,force=False): |
---|
53 | if self.config == None or force: |
---|
54 | self.config = configparser.ConfigParser(delimiters=":") |
---|
55 | self.config.optionxform = str |
---|
56 | self.config.read(self.root + "/update.cfg") |
---|
57 | |
---|
58 | def saveConfig(self): |
---|
59 | f = open(self.root + "/update.cfg",'w') |
---|
60 | self.config.write(f) |
---|
61 | f.close() |
---|
62 | |
---|
63 | def downloadFile(self,orig,dest): |
---|
64 | openwebsite = urllib.request.urlopen(orig) |
---|
65 | soup = BeautifulSoup(openwebsite,'html.parser') |
---|
66 | if orig.endswith('/'): |
---|
67 | orig = orig[:-1] |
---|
68 | filename = orig.rsplit('/',1)[-1] |
---|
69 | blacklistlinks = ['description','parent directory','size','last modified','name','doc/'] |
---|
70 | if len(soup.findAll('html')) > 0: |
---|
71 | # Folder |
---|
72 | folder = dest + "/" + filename |
---|
73 | try: |
---|
74 | os.mkdir(folder) |
---|
75 | except Exception as e: |
---|
76 | pass |
---|
77 | for link in soup.findAll('a'): |
---|
78 | if str(link.text).lower() not in blacklistlinks: |
---|
79 | self.downloadFile(orig+"/"+str(link.get('href')),folder) |
---|
80 | else: |
---|
81 | urllib.request.urlretrieve(orig,dest + "/" + filename) |
---|
82 | sys.stdout.write('.') |
---|
83 | sys.stdout.flush() |
---|
84 | |
---|
85 | |
---|
86 | def renameSeeds(self,flavour,tempfolder): |
---|
87 | baseflavour = flavour.rsplit('.',1)[0] |
---|
88 | basedir = os.path.join(tempfolder,flavour) |
---|
89 | structurefile = os.path.join(basedir,'STRUCTURE') |
---|
90 | seedstorename = [] |
---|
91 | |
---|
92 | if not os.path.exists(structurefile): |
---|
93 | return 0 |
---|
94 | |
---|
95 | #get only seeds exists |
---|
96 | dfstructure = open(structurefile,'r') |
---|
97 | lines = dfstructure.readlines() |
---|
98 | dfstructure.close() |
---|
99 | for line in lines: |
---|
100 | seeds = line.split(":") |
---|
101 | if len(seeds) <= 1: |
---|
102 | continue |
---|
103 | if os.path.exists(os.path.join(basedir,seeds[0])): |
---|
104 | seedstorename.append(seeds[0]) |
---|
105 | |
---|
106 | for seed in seedstorename: |
---|
107 | pathseed = os.path.join(basedir,seed) |
---|
108 | dfstructure = fileinput.FileInput(pathseed,inplace=True) |
---|
109 | for line in dfstructure: |
---|
110 | if re.search(r'^Task:Seeds:',line,re.I): |
---|
111 | for seed in seedstorename: |
---|
112 | line = re.sub(r'(Task-Seeds:)((\s*\w+\s+)*)('+seed+r')(\s+.*|$)',r'\1\2'+baseflavour+r'-\4\5',line,0,re.I) |
---|
113 | print(line,end="") |
---|
114 | dfstructure.close() |
---|
115 | os.rename(pathseed ,os.path.join(basedir,baseflavour+"-"+seed)) |
---|
116 | |
---|
117 | dfstructure = fileinput.FileInput(structurefile,inplace=True) |
---|
118 | for line in dfstructure: |
---|
119 | if len(line.split(":")) <= 1: |
---|
120 | continue |
---|
121 | for seed in seedstorename: |
---|
122 | line = re.sub(r"^"+seed+":",baseflavour+"-"+seed+":",line) |
---|
123 | line = re.sub(r"(\s*)"+seed+r"(\s+|$)",r"\1"+baseflavour+"-"+seed+r"\2",line) |
---|
124 | print(line,end="") |
---|
125 | shutil.move(basedir,self.root + "/seeds/") |
---|
126 | |
---|
127 | def complete_create(self,text,line,begidx,endidx): |
---|
128 | if len(self.flavours) == 0: |
---|
129 | website = "http://people.canonical.com/~ubuntu-archive/seeds/" |
---|
130 | openwebsite = urllib.request.urlopen(website) |
---|
131 | soup = BeautifulSoup(openwebsite,'html.parser') |
---|
132 | links = soup.findAll('a')[4:] |
---|
133 | for link in links: |
---|
134 | folder = str(link.text)[:-1] |
---|
135 | if not folder.startswith('platform'): |
---|
136 | self.flavours.append(folder) |
---|
137 | return [i for i in self.flavours if i.startswith(text)] |
---|
138 | |
---|
139 | def downloadPlatformSeed(self,flavour): |
---|
140 | base = flavour.rsplit('.',1)[-1] |
---|
141 | platformurl = "http://people.canonical.com/~ubuntu-archive/seeds/platform."+base |
---|
142 | print("Downloading platform " + base) |
---|
143 | self.downloadFile(platformurl,self.root + "/seeds/") |
---|
144 | print("") |
---|
145 | |
---|
146 | def downloadFlavourSeed(self,flavour): |
---|
147 | flavoururl = "http://people.canonical.com/~ubuntu-archive/seeds/"+flavour |
---|
148 | tempfolder = tempfile.mkdtemp() |
---|
149 | print("Downloading flavour " + flavour) |
---|
150 | self.downloadFile(flavoururl,tempfolder) |
---|
151 | print("Done") |
---|
152 | print("Renaming seeds") |
---|
153 | self.renameSeeds(flavour,tempfolder) |
---|
154 | |
---|
155 | def downloadSeeds(self,flavour): |
---|
156 | self.downloadPlatformSeed(flavour) |
---|
157 | self.downloadFlavourSeed(flavour) |
---|
158 | |
---|
159 | def createNeededStructure(self,codename): |
---|
160 | try: |
---|
161 | os.mkdir(self.root + "/seeds") |
---|
162 | os.mkdir(self.root + "/seeds/lliurex") |
---|
163 | except: |
---|
164 | pass |
---|
165 | f = open(self.root + "/update.cfg",'w') |
---|
166 | f.write("[DEFAULT]\n") |
---|
167 | f.write("dist: "+codename+"\n\n") |
---|
168 | f.write("["+codename+"]\n") |
---|
169 | f.write("seeds:\n") |
---|
170 | f.write("architectures: i386 amd64\n") |
---|
171 | f.write("seed_base: seeds \n") |
---|
172 | f.write("seed_dist: lliurex \n") |
---|
173 | f.write("archive_base/default: http://archive.ubuntu.com/ubuntu http://ppa.launchpad.net/llxdev/"+codename+"/ubuntu\n") |
---|
174 | f.write("components: main restricted universe multiverse\n") |
---|
175 | f.close() |
---|
176 | self.loadConfig(True) |
---|
177 | |
---|
178 | |
---|
179 | def newOutputSeeds(self): |
---|
180 | defaultOutSeeds = ["cdd-live","cdd-edu-gdesktop","cdd-ltsp-server","cdd-gdesktop","cdd-supported","cdd-net-gserver","cdd-edu-class-gclient","cdd-ltsp-net-gserver","cdd-edu-class-gserver","cdd-network-client-promo","cdd-edu-music-gdesktop","cdd-edu-infantil-gdesktop","cdd-xdesktop","cdd-xdesktop-extended","cdd-gdesktop-gva","cdd-edu-gserver-extra","cdd-devel","cdd-gdesktop-pime","cdd-minimal"] |
---|
181 | answer = input("Use default seeds?([y]/n): ").lower() |
---|
182 | if answer == "" or answer == "y" or answer == "yes": |
---|
183 | dist = self.config.get("DEFAULT","dist") |
---|
184 | self.config.set(dist,"seeds"," ".join(defaultOutSeeds)) |
---|
185 | self.saveConfig() |
---|
186 | |
---|
187 | def loadSeeds(self): |
---|
188 | self.seeds = {} |
---|
189 | for (dirpath,dirname,filelist) in os.walk(self.root+"/seeds"): |
---|
190 | for seed in filelist: |
---|
191 | if seed.lower() != "structure" and dirpath != self.root+"/seeds" : |
---|
192 | self.seeds[seed] = os.path.basename(dirpath) + "/" + seed |
---|
193 | |
---|
194 | |
---|
195 | def ensureOutputSeeds(self): |
---|
196 | dist = self.config.get("DEFAULT","dist") |
---|
197 | outputseeds = self.config.get(dist,"seeds") |
---|
198 | for seed in self.config.get(dist,"seeds").strip().split(" "): |
---|
199 | if not seed in self.seeds and seed != "": |
---|
200 | seedpath = self.root + "/seeds/lliurex/"+ seed |
---|
201 | f = open(seedpath,'w') |
---|
202 | f.close() |
---|
203 | self.seeds[seed] = seedpath |
---|
204 | |
---|
205 | def ensureDebianPackage(self): |
---|
206 | if not os.path.exists('debian'): |
---|
207 | answer = input("Do you want create debian folder?([y]/n): ").lower() |
---|
208 | if answer == "" or answer == "y" or answer == "yes": |
---|
209 | subprocess.call(["dh_make","-s","-n","-p","lliurex-meta_0.1"]) |
---|
210 | |
---|
211 | |
---|
212 | def do_create(self,line): |
---|
213 | 'create UbuntuFlavour [LliureXCodeName]' |
---|
214 | args = line.split(" ") |
---|
215 | lliurexcodename = args[1] if len(args) > 1 else args[0].rsplit('.',1)[-1] |
---|
216 | self.createNeededStructure(lliurexcodename) |
---|
217 | self.downloadSeeds(args[0]) |
---|
218 | self.newOutputSeeds() |
---|
219 | self.ensureOutputSeeds() |
---|
220 | self.ensureDebianPackage() |
---|
221 | |
---|
222 | def printDepends(self,id,tabs,parent=None): |
---|
223 | depends = self.structure[id] |
---|
224 | parentstr = " ── ( " +str(parent)+" )" if parent != None else "" |
---|
225 | print(" "*tabs + str(id) + parentstr) |
---|
226 | for depend in depends: |
---|
227 | self.printDepends(depend,tabs+1,id) |
---|
228 | |
---|
229 | |
---|
230 | def complete_structurePrint(self,text,line,begidx,endidx): |
---|
231 | self.loadStructure() |
---|
232 | return [i for i in self.structure.keys() if i.startswith(text)] |
---|
233 | |
---|
234 | def do_structurePrint(self,line): |
---|
235 | self.loadStructure() |
---|
236 | #import json |
---|
237 | #print(json.dumps(self.structure,indent=4)) |
---|
238 | print("") |
---|
239 | if line != "": |
---|
240 | self.printDepends(line,0) |
---|
241 | else: |
---|
242 | for key in self.structure.keys(): |
---|
243 | self.printDepends(key,0) |
---|
244 | |
---|
245 | def complete_seedsRdepends(self,text,line,begidx,endidx): |
---|
246 | self.loadStructure() |
---|
247 | return [i for i in self.structure.keys() if i.startswith(text)] |
---|
248 | |
---|
249 | # def printRdepends(self,id,tabs,parent=None): |
---|
250 | # found = False |
---|
251 | # for seed in self.structure.keys(): |
---|
252 | # if id in self.structure[seed]: |
---|
253 | # found = True |
---|
254 | # parentstr = " ── ( " +str(parent)+" )" if parent != None else "" |
---|
255 | # print(" "*(10 - tabs) + str(seed) + parentstr) |
---|
256 | # self.printRdepends(seed,tabs+1,seed) |
---|
257 | # if found: |
---|
258 | # print("") |
---|
259 | |
---|
260 | def searchRdepends(self,id): |
---|
261 | result = [] |
---|
262 | for seed in self.structure.keys(): |
---|
263 | if id in self.structure[seed]: |
---|
264 | result.append(seed) |
---|
265 | result += self.searchRdepends(seed) |
---|
266 | return result |
---|
267 | |
---|
268 | def printRdepends(self,id,tabs,listToPrint,needle,parent=None): |
---|
269 | if id in listToPrint: |
---|
270 | depends = self.structure[id] |
---|
271 | parentstr = " ── ( " +str(parent)+" )" if parent != None else "" |
---|
272 | if not needle in depends: |
---|
273 | print(" "*tabs + str(id) + parentstr) |
---|
274 | for depend in depends: |
---|
275 | self.printRdepends(depend,tabs+1,listToPrint,id) |
---|
276 | |
---|
277 | def editValuesConfig(self,section,value,editor="vim"): |
---|
278 | if editor == "": |
---|
279 | editor = "vim" |
---|
280 | options = self.config.get(section,value).split("#") |
---|
281 | enabled = options[0].strip().split(" ") |
---|
282 | disabled = options[1].strip().split(" ") if len(options) > 1 else [] |
---|
283 | tempfile = '/tmp/.lliurex-meta-maker.tmp' |
---|
284 | f = open(tempfile,'w') |
---|
285 | f.write("############### "+value+" #######################\n") |
---|
286 | f.write("#enabled values\n") |
---|
287 | for en in enabled: |
---|
288 | f.write(en + "\n") |
---|
289 | f.write("\n#disabled values\n") |
---|
290 | for di in disabled: |
---|
291 | f.write(di + "\n") |
---|
292 | f.close() |
---|
293 | subprocess.call([editor,tempfile]) |
---|
294 | f = open(tempfile,'r') |
---|
295 | lines = f.readlines() |
---|
296 | enabled = [] |
---|
297 | disabled = [] |
---|
298 | inenable = True |
---|
299 | for auxline in lines: |
---|
300 | if auxline.lower().startswith("#enabled"): |
---|
301 | inenable = True |
---|
302 | continue |
---|
303 | if auxline.lower().startswith("#disabled"): |
---|
304 | inenable = False |
---|
305 | continue |
---|
306 | if auxline.startswith("#"): |
---|
307 | continue |
---|
308 | if inenable: |
---|
309 | enabled.append(auxline.strip()) |
---|
310 | else: |
---|
311 | disabled.append(auxline.strip()) |
---|
312 | f.close() |
---|
313 | appenddisabled = "#"+" ".join(disabled) if len(disabled) > 0 else "" |
---|
314 | finalvalue = " ".join(enabled) + appenddisabled |
---|
315 | self.config.set(section,value,finalvalue) |
---|
316 | |
---|
317 | |
---|
318 | def do_seedsRdepends(self,line): |
---|
319 | self.loadStructure() |
---|
320 | listToPrint = self.searchRdepends(line) |
---|
321 | for key in self.structure.keys(): |
---|
322 | self.printRdepends(key,0,listToPrint,line) |
---|
323 | |
---|
324 | def do_archiveBaseUpdate(self,line): |
---|
325 | self.loadConfig() |
---|
326 | dist = self.config.get("DEFAULT","dist") |
---|
327 | listoptions = self.config.items(dist) |
---|
328 | resultitems = [] |
---|
329 | for x in listoptions: |
---|
330 | if x[0].startswith('archive_base'): |
---|
331 | resultitems.append(x[0]) |
---|
332 | for x in resultitems: |
---|
333 | self.editValuesConfig(dist,x,line.strip()) |
---|
334 | self.saveConfig() |
---|
335 | |
---|
336 | def complete_seedEdit(self,text,line,begidx,endidx): |
---|
337 | self.loadSeeds() |
---|
338 | return [i for i in self.seeds if i.startswith(text)] |
---|
339 | |
---|
340 | def do_seedEdit(self,line): |
---|
341 | self.loadSeeds() |
---|
342 | subprocess.call(['vim','seeds/'+self.seeds[line]]) |
---|
343 | |
---|
344 | def do_seedCreate(self,line): |
---|
345 | self.loadSeeds() |
---|
346 | seedname = line.strip() |
---|
347 | if seedname in self.seeds.keys(): |
---|
348 | print("\n Error : seed " + seedname + " already exists \n") |
---|
349 | else: |
---|
350 | f = open('seeds/lliurex/'+seedname,'w') |
---|
351 | f.close() |
---|
352 | subprocess.call(['vim','seeds/lliurex/'+seedname]) |
---|
353 | |
---|
354 | def do_seedSearchPackages(self,line): |
---|
355 | self.loadSeeds() |
---|
356 | packagelist = line.split(" ") |
---|
357 | listfinds = {k:[] for k in packagelist} |
---|
358 | for seed in self.seeds: |
---|
359 | f = open('seeds/'+self.seeds[seed],'r+b') |
---|
360 | try: |
---|
361 | mapfile = mmap.mmap(f.fileno(),0) |
---|
362 | except Exception as e: |
---|
363 | continue |
---|
364 | print('seeds/'+self.seeds[seed]) |
---|
365 | for needle in packagelist: |
---|
366 | regex = '\*\s*' + needle + '\s*\Z' |
---|
367 | found = re.search(regex.encode(),mapfile) |
---|
368 | if found != None: |
---|
369 | listfinds[needle].append(seed) |
---|
370 | print("Package \t Seed") |
---|
371 | print("======= \t =====") |
---|
372 | for seed in listfinds.keys(): |
---|
373 | print(seed,"\t",listfinds[seed]) |
---|
374 | |
---|
375 | def complete_structureEdit(self,text,line,begidx,endidx): |
---|
376 | folders = [d for d in os.listdir('seeds') if os.path.isdir(os.path.join('seeds', d))] |
---|
377 | return [i for i in folders if i.startswith(text)] |
---|
378 | |
---|
379 | def do_structureEdit(self,line): |
---|
380 | folder = line.strip() |
---|
381 | subprocess.call(['vim','seeds/'+folder+'/STRUCTURE']) |
---|
382 | |
---|
383 | def do_update(self,line): |
---|
384 | pass |
---|
385 | |
---|
386 | |
---|
387 | def do_exit(self,line): |
---|
388 | 'Exit' |
---|
389 | return True |
---|
390 | |
---|
391 | def do_EOF(self, line): |
---|
392 | 'Exit' |
---|
393 | return True |
---|