1 | import subprocess |
---|
2 | import cmd |
---|
3 | import configparser |
---|
4 | import os |
---|
5 | import shutil |
---|
6 | import urllib |
---|
7 | import urllib.request |
---|
8 | from bs4 import BeautifulSoup |
---|
9 | import sys |
---|
10 | import mmap |
---|
11 | import re |
---|
12 | |
---|
13 | class MetaMaker(cmd.Cmd): |
---|
14 | intro = "Welcome to Lliurex Meta Maker \n" |
---|
15 | prompt = 'LlxMeta: ' |
---|
16 | flavours = [] |
---|
17 | root = os.getcwd() |
---|
18 | config = None |
---|
19 | seeds = {} |
---|
20 | structure = {} |
---|
21 | |
---|
22 | def completenames(self,text,*ignores): |
---|
23 | lst = cmd.Cmd.completenames(self,text,*ignores) |
---|
24 | return [a for a in lst if a != 'EOF'] |
---|
25 | |
---|
26 | def loadStructure(self): |
---|
27 | structurefiles = [] |
---|
28 | for (dirpath,dirname,filelist) in os.walk(self.root+"/seeds"): |
---|
29 | for seed in filelist: |
---|
30 | if seed.lower() == "structure": |
---|
31 | structurefiles.append(dirpath + "/" + seed) |
---|
32 | for strucfilepath in structurefiles: |
---|
33 | fd = open(strucfilepath,'r') |
---|
34 | content = list(map(str.rstrip,fd.readlines())) |
---|
35 | for line in content: |
---|
36 | seed = line.split(":") |
---|
37 | if len(seed) > 1: |
---|
38 | depends = seed[1].lstrip().split(" ") |
---|
39 | self.structure[seed[0]] = depends if depends != [""] else [] |
---|
40 | |
---|
41 | def loadConfig(self,force=False): |
---|
42 | if self.config == None or force: |
---|
43 | self.config = configparser.ConfigParser(delimiters=":") |
---|
44 | self.config.optionxform = str |
---|
45 | self.config.read(self.root + "/update.cfg") |
---|
46 | |
---|
47 | def saveConfig(self): |
---|
48 | f = open(self.root + "/update.cfg",'w') |
---|
49 | self.config.write(f) |
---|
50 | f.close() |
---|
51 | |
---|
52 | def downloadFile(self,orig,dest): |
---|
53 | openwebsite = urllib.request.urlopen(orig) |
---|
54 | soup = BeautifulSoup(openwebsite,'html.parser') |
---|
55 | if orig.endswith('/'): |
---|
56 | orig = orig[:-1] |
---|
57 | filename = orig.rsplit('/',1)[-1] |
---|
58 | blacklistlinks = ['description','parent directory','size','last modified','name','doc/'] |
---|
59 | if len(soup.findAll('html')) > 0: |
---|
60 | # Folder |
---|
61 | folder = dest + "/" + filename |
---|
62 | try: |
---|
63 | os.mkdir(folder) |
---|
64 | except Exception as e: |
---|
65 | pass |
---|
66 | for link in soup.findAll('a'): |
---|
67 | if str(link.text).lower() not in blacklistlinks: |
---|
68 | self.downloadFile(orig+"/"+str(link.get('href')),folder) |
---|
69 | else: |
---|
70 | urllib.request.urlretrieve(orig,dest + "/" + filename) |
---|
71 | sys.stdout.write('.') |
---|
72 | sys.stdout.flush() |
---|
73 | |
---|
74 | def complete_create(self,text,line,begidx,endidx): |
---|
75 | if len(self.flavours) == 0: |
---|
76 | website = "http://people.canonical.com/~ubuntu-archive/seeds/" |
---|
77 | openwebsite = urllib.request.urlopen(website) |
---|
78 | soup = BeautifulSoup(openwebsite) |
---|
79 | links = soup.findAll('a')[4:] |
---|
80 | for link in links: |
---|
81 | folder = str(link.text)[:-1] |
---|
82 | if not folder.startswith('platform'): |
---|
83 | self.flavours.append(folder) |
---|
84 | return [i for i in self.flavours if i.startswith(text)] |
---|
85 | |
---|
86 | def downloadSeeds(self,flavour): |
---|
87 | base = flavour.rsplit('.',1)[-1] |
---|
88 | originseeds = ["http://people.canonical.com/~ubuntu-archive/seeds/"+flavour,"http://people.canonical.com/~ubuntu-archive/seeds/platform."+base] |
---|
89 | for urlseed in originseeds : |
---|
90 | try: |
---|
91 | print("Downloading " + urlseed) |
---|
92 | self.downloadFile(urlseed,self.root + "/seeds/") |
---|
93 | print("") |
---|
94 | except Exception as e: |
---|
95 | print(e) |
---|
96 | |
---|
97 | def createNeededStructure(self,codename): |
---|
98 | try: |
---|
99 | os.mkdir(self.root + "/seeds") |
---|
100 | os.mkdir(self.root + "/seeds/lliurex") |
---|
101 | except: |
---|
102 | pass |
---|
103 | f = open(self.root + "/update.cfg",'w') |
---|
104 | f.write("[DEFAULT]\n") |
---|
105 | f.write("dist: "+codename+"\n\n") |
---|
106 | f.write("["+codename+"]\n") |
---|
107 | f.write("seeds:\n") |
---|
108 | f.write("output_seeds:\n") |
---|
109 | f.write("architectures: i386 amd64\n") |
---|
110 | f.write("seed_base: "+self.root+"/seeds \n") |
---|
111 | f.write("archive_base/default: \n") |
---|
112 | f.write("components: main restricted universe multiverse\n") |
---|
113 | f.close() |
---|
114 | self.loadConfig(True) |
---|
115 | |
---|
116 | |
---|
117 | def newOutputSeeds(self): |
---|
118 | defaultOutSeeds = ["cdd-live","cdd-edu-gdesktop","cdd-ltsp-server","cdd-gdesktop","cdd-supported","cdd-net-gserver","cdd-edu-class-gclient","cdd-ltsp-net-gserver","cdd-edu-class-gserver","cdd-network-client-promo","cdd-edu-music-gdesktop","cdd-edu-infantil-gdesktop","cdd-xdesktop","cdd-xdesktop-extended","cdd-gdesktop-gva","cdd-edu-gserver-extra","cdd-devel","cdd-gdesktop-pime","cdd-minimal"] |
---|
119 | answer = input("Use default seeds?([y]/n): ").lower() |
---|
120 | if answer == "" or answer == "y" or answer == "yes": |
---|
121 | dist = self.config.get("DEFAULT","dist") |
---|
122 | self.config.set(dist,"output_seeds"," ".join(defaultOutSeeds)) |
---|
123 | self.saveConfig() |
---|
124 | |
---|
125 | def loadSeeds(self): |
---|
126 | self.seeds = {} |
---|
127 | for (dirpath,dirname,filelist) in os.walk(self.root+"/seeds"): |
---|
128 | for seed in filelist: |
---|
129 | if seed.lower() != "structure" and dirpath != self.root+"/seeds" : |
---|
130 | self.seeds[seed] = os.path.basename(dirpath) + "/" + seed |
---|
131 | |
---|
132 | |
---|
133 | def ensureOutputSeeds(self): |
---|
134 | dist = self.config.get("DEFAULT","dist") |
---|
135 | outputseeds = self.config.get(dist,"output_seeds") |
---|
136 | for seed in self.config.get(dist,"output_seeds").strip().split(" "): |
---|
137 | if not seed in self.seeds and seed != "": |
---|
138 | seedpath = self.root + "/seeds/lliurex/"+ seed |
---|
139 | f = open(seedpath,'w') |
---|
140 | f.close() |
---|
141 | self.seeds[seed] = seedpath |
---|
142 | |
---|
143 | def ensureDebianPackage(self): |
---|
144 | if not os.path.exists('debian'): |
---|
145 | answer = input("Do you want create debian folder?([y]/n): ").lower() |
---|
146 | if answer == "" or answer == "y" or answer == "yes": |
---|
147 | subprocess.call(["dh_make","-s","-n","-p","lliurex-meta_0.1"]) |
---|
148 | |
---|
149 | |
---|
150 | def do_create(self,line): |
---|
151 | 'create UbuntuFlavour [LliureXCodeName]' |
---|
152 | args = line.split(" ") |
---|
153 | lliurexcodename = args[1] if len(args) > 1 else args[0].rsplit('.',1)[-1] |
---|
154 | self.createNeededStructure(lliurexcodename) |
---|
155 | self.downloadSeeds(args[0]) |
---|
156 | self.newOutputSeeds() |
---|
157 | self.ensureOutputSeeds() |
---|
158 | self.ensureDebianPackage() |
---|
159 | |
---|
160 | def printDepends(self,id,tabs,parent=None): |
---|
161 | depends = self.structure[id] |
---|
162 | parentstr = " ── ( " +str(parent)+" )" if parent != None else "" |
---|
163 | print(" "*tabs + str(id) + parentstr) |
---|
164 | for depend in depends: |
---|
165 | self.printDepends(depend,tabs+1,id) |
---|
166 | |
---|
167 | |
---|
168 | def complete_structurePrint(self,text,line,begidx,endidx): |
---|
169 | self.loadStructure() |
---|
170 | return [i for i in self.structure.keys() if i.startswith(text)] |
---|
171 | |
---|
172 | def do_structurePrint(self,line): |
---|
173 | self.loadStructure() |
---|
174 | #import json |
---|
175 | #print(json.dumps(self.structure,indent=4)) |
---|
176 | print("") |
---|
177 | if line != "": |
---|
178 | self.printDepends(line,0) |
---|
179 | else: |
---|
180 | for key in self.structure.keys(): |
---|
181 | self.printDepends(key,0) |
---|
182 | |
---|
183 | def complete_seedsRdepends(self,text,line,begidx,endidx): |
---|
184 | self.loadStructure() |
---|
185 | return [i for i in self.structure.keys() if i.startswith(text)] |
---|
186 | |
---|
187 | # def printRdepends(self,id,tabs,parent=None): |
---|
188 | # found = False |
---|
189 | # for seed in self.structure.keys(): |
---|
190 | # if id in self.structure[seed]: |
---|
191 | # found = True |
---|
192 | # parentstr = " ── ( " +str(parent)+" )" if parent != None else "" |
---|
193 | # print(" "*(10 - tabs) + str(seed) + parentstr) |
---|
194 | # self.printRdepends(seed,tabs+1,seed) |
---|
195 | # if found: |
---|
196 | # print("") |
---|
197 | |
---|
198 | def searchRdepends(self,id): |
---|
199 | result = [] |
---|
200 | for seed in self.structure.keys(): |
---|
201 | if id in self.structure[seed]: |
---|
202 | result.append(seed) |
---|
203 | result += self.searchRdepends(seed) |
---|
204 | return result |
---|
205 | |
---|
206 | def printRdepends(self,id,tabs,listToPrint,needle,parent=None): |
---|
207 | if id in listToPrint: |
---|
208 | depends = self.structure[id] |
---|
209 | parentstr = " ── ( " +str(parent)+" )" if parent != None else "" |
---|
210 | if not needle in depends: |
---|
211 | print(" "*tabs + str(id) + parentstr) |
---|
212 | for depend in depends: |
---|
213 | self.printRdepends(depend,tabs+1,listToPrint,id) |
---|
214 | |
---|
215 | def editValuesConfig(self,section,value,editor="vim"): |
---|
216 | if editor == "": |
---|
217 | editor = "vim" |
---|
218 | options = self.config.get(section,value).split("#") |
---|
219 | enabled = options[0].strip().split(" ") |
---|
220 | disabled = options[1].strip().split(" ") if len(options) > 1 else [] |
---|
221 | tempfile = '/tmp/.lliurex-meta-maker.tmp' |
---|
222 | f = open(tempfile,'w') |
---|
223 | f.write("############### "+value+" #######################\n") |
---|
224 | f.write("#enabled values\n") |
---|
225 | for en in enabled: |
---|
226 | f.write(en + "\n") |
---|
227 | f.write("\n#disabled values\n") |
---|
228 | for di in disabled: |
---|
229 | f.write(di + "\n") |
---|
230 | f.close() |
---|
231 | subprocess.call([editor,tempfile]) |
---|
232 | f = open(tempfile,'r') |
---|
233 | lines = f.readlines() |
---|
234 | enabled = [] |
---|
235 | disabled = [] |
---|
236 | inenable = True |
---|
237 | for auxline in lines: |
---|
238 | if auxline.lower().startswith("#enabled"): |
---|
239 | inenable = True |
---|
240 | continue |
---|
241 | if auxline.lower().startswith("#disabled"): |
---|
242 | inenable = False |
---|
243 | continue |
---|
244 | if auxline.startswith("#"): |
---|
245 | continue |
---|
246 | if inenable: |
---|
247 | enabled.append(auxline.strip()) |
---|
248 | else: |
---|
249 | disabled.append(auxline.strip()) |
---|
250 | f.close() |
---|
251 | appenddisabled = "#"+" ".join(disabled) if len(disabled) > 0 else "" |
---|
252 | finalvalue = " ".join(enabled) + appenddisabled |
---|
253 | self.config.set(section,value,finalvalue) |
---|
254 | |
---|
255 | |
---|
256 | def do_seedsRdepends(self,line): |
---|
257 | self.loadStructure() |
---|
258 | listToPrint = self.searchRdepends(line) |
---|
259 | for key in self.structure.keys(): |
---|
260 | self.printRdepends(key,0,listToPrint,line) |
---|
261 | |
---|
262 | def do_archiveBaseUpdate(self,line): |
---|
263 | self.loadConfig() |
---|
264 | dist = self.config.get("DEFAULT","dist") |
---|
265 | listoptions = self.config.items(dist) |
---|
266 | resultitems = [] |
---|
267 | for x in listoptions: |
---|
268 | if x[0].startswith('archive_base'): |
---|
269 | resultitems.append(x[0]) |
---|
270 | for x in resultitems: |
---|
271 | self.editValuesConfig(dist,x,line.strip()) |
---|
272 | self.saveConfig() |
---|
273 | |
---|
274 | def complete_seedEdit(self,text,line,begidx,endidx): |
---|
275 | self.loadSeeds() |
---|
276 | return [i for i in self.seeds if i.startswith(text)] |
---|
277 | |
---|
278 | def do_seedEdit(self,line): |
---|
279 | self.loadSeeds() |
---|
280 | subprocess.call(['vim','seeds/'+self.seeds[line]]) |
---|
281 | |
---|
282 | def do_seedCreate(self,line): |
---|
283 | self.loadSeeds() |
---|
284 | seedname = line.strip() |
---|
285 | if seedname in self.seeds.keys(): |
---|
286 | print("\n Error : seed " + seedname + " already exists \n") |
---|
287 | else: |
---|
288 | f = open('seeds/lliurex/'+seedname,'w') |
---|
289 | f.close() |
---|
290 | subprocess.call(['vim','seeds/lliurex/'+seedname]) |
---|
291 | |
---|
292 | def do_seedSearchPackages(self,line): |
---|
293 | self.loadSeeds() |
---|
294 | packagelist = line.split(" ") |
---|
295 | listfinds = {k:[] for k in packagelist} |
---|
296 | for seed in self.seeds: |
---|
297 | f = open('seeds/'+self.seeds[seed],'r+b') |
---|
298 | try: |
---|
299 | mapfile = mmap.mmap(f.fileno(),0) |
---|
300 | except Exception as e: |
---|
301 | continue |
---|
302 | print('seeds/'+self.seeds[seed]) |
---|
303 | for needle in packagelist: |
---|
304 | regex = '\*\s*' + needle + '\s*\Z' |
---|
305 | found = re.search(regex.encode(),mapfile) |
---|
306 | if found != None: |
---|
307 | listfinds[needle].append(seed) |
---|
308 | print("Package \t Seed") |
---|
309 | print("======= \t =====") |
---|
310 | for seed in listfinds.keys(): |
---|
311 | print(seed,"\t",listfinds[seed]) |
---|
312 | |
---|
313 | def complete_structureEdit(self,text,line,begidx,endidx): |
---|
314 | folders = [d for d in os.listdir('seeds') if os.path.isdir(os.path.join('seeds', d))] |
---|
315 | return [i for i in folders if i.startswith(text)] |
---|
316 | |
---|
317 | def do_structureEdit(self,line): |
---|
318 | folder = line.strip() |
---|
319 | subprocess.call(['vim','seeds/'+folder+'/STRUCTURE']) |
---|
320 | |
---|
321 | def do_update(self,line): |
---|
322 | pass |
---|
323 | |
---|
324 | |
---|
325 | def do_exit(self,line): |
---|
326 | 'Exit' |
---|
327 | return True |
---|
328 | |
---|
329 | def do_EOF(self, line): |
---|
330 | 'Exit' |
---|
331 | return True |
---|