Changeset 7069
- Timestamp:
- Mar 16, 2018, 1:21:09 PM (3 years ago)
- Location:
- lliurex-store/trunk/fuentes/python3-lliurex-store.install/usr/share/lliurexstore/plugins
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
lliurex-store/trunk/fuentes/python3-lliurex-store.install/usr/share/lliurexstore/plugins/appImageManager.py
r7053 r7069 1 1 #The name of the main class must match the file name in lowercase 2 2 import urllib 3 from urllib.request import Request 4 from urllib.request import urlretrieve 3 5 import shutil 4 6 import json … … 24 26 self.result['data']={} 25 27 self.result['status']={} 26 # self.bundles_dir="/var/lib/lliurexstore/bundles" 27 self.conf_dir=os.getenv("HOME")+"/.lliurexstore" 28 self.conf_dir=os.getenv("HOME")+"/.cache/lliurex-store" 28 29 self.bundles_dir=self.conf_dir+"/bundles" 29 30 self.bundle_types=['appimg'] 30 # self.appimage_dir='/opt/bundles/appimg'31 31 self.appimage_dir=self.conf_dir+"/appimg" 32 self.repository_url='https://dl.bintray.com/probono/AppImages' 33 #To get the description of an app we must go to a specific url. 32 #To get the description of an app we must go to a specific url defined in url_info. 34 33 #$(appname) we'll be replaced with the appname so the url matches the right one. 35 34 #If other site has other url naming convention it'll be mandatory to define it with the appropiate replacements 36 self.info_url='https://bintray.com/probono/AppImages/$(appname)' 35 self.repos={'probono':{'url':'https://dl.bintray.com/probono/AppImages', 'url_info':'https://bintray.com/probono/AppImages/$(appname)'}} 36 #Appimges not stored in a repo must be listed here, providing the download url and the info url (if there's any) 37 self.external_appimages={'synfig64':{'url':'https://kent.dl.sourceforge.net/project/synfig/releases/1.2.1/linux/SynfigStudio-1.2.1-64bit.appimage','url_info':''},'synfig32':{'url':'https://kent.dl.sourceforge.net/project/synfig/releases/1.2.1/linux/SynfigStudio-1.2.1-32bit.appimage','url_info':''}} 37 38 self.disabled=False 38 39 self.count=0 … … 113 114 def _install_appimage(self,app_info): 114 115 app_info=self._get_info(app_info) 116 self._debug("Installing %s"%app_info) 115 117 if app_info['state']=='installed': 116 118 self._set_status(4) 117 119 else: 118 appimage_url=self.repository_url+'/'+app_info['package'] 120 # appimage_url=self.repository_url+'/'+app_info['package'] 121 appimage_url=app_info['homepage']+'/'+app_info['package'] 119 122 self._debug("Downloading "+appimage_url) 120 123 dest_path=self.appimage_dir+'/'+app_info['package'] 121 124 if appimage_url: 122 125 try: 123 with urllib.request.urlopen(appimage_url) as response, open(dest_path, 'wb') as out_file: 126 req=Request(appimage_url, headers={'User-Agent':'Mozilla/5.0'}) 127 with urllib.request.urlopen(req) as response, open(dest_path, 'wb') as out_file: 124 128 bf=16*1024 125 129 acumbf=0 130 # print("Response: %s"%response.info()) 126 131 app_size=int(response.info()['Content-Length']) 132 # print("APP SIZE: %s"%app_size) 127 133 while True: 128 134 if acumbf>=app_size: … … 134 140 os.chmod(dest_path, st.st_mode | 0o111) 135 141 self._set_status(0) 136 except: 142 except Exception as e: 143 print(e) 137 144 self._set_status(5) 138 145 else: … … 161 168 app_info['state']='installed' 162 169 #Get size 163 appimage_url= self.repository_url+'/'+app_info['package']170 appimage_url=app_info['homepage']+'/'+app_info['package'] 164 171 dest_path=self.appimage_dir+'/'+app_info['package'] 165 172 if appimage_url: … … 206 213 applist=[] 207 214 progress_bar="#" 208 repositories_list={'appimg':['https://dl.bintray.com/probono/AppImages']}209 #For get the description of an app we must go to a specific url.210 #$(appname) we'll be replaced with the appname so the url matches the right one.211 #If other site has other url naming convention it'll be mandatory to define it with the appropiate replacements212 info_list={'appimg':'https://bintray.com/probono/AppImages/$(appname)'}213 215 self.descriptions_dict={} 214 215 for repo_type,repo_types in repositories_list.items():216 info_url=info_list[repo_type]217 outdir=self.bundles_dir+'/'+repo_type+'/'216 all_apps=[] 217 outdir=self.bundles_dir+'/appimg/' 218 #Load repos 219 for repo_name,repo_info in self.repos.items(): 218 220 if self._chk_bundle_dir(outdir): 219 for repo in repo_types: 220 self._debug(("Fetching repo %s")%(repo)) 221 # print (("Fetching %s catalogue: "+progress_bar)%repo_type,end="\r") 222 # progress_bar=progress_bar+"#" 223 # print (("Fetching %s catalogue: "+progress_bar)%repo_type,end="\r") 224 applist=self._generate_applist(self._fetch_repo(repo)) 225 # progress_bar=progress_bar+"##" 226 # print (("Fetching %s catalogue: "+progress_bar)%repo_type,end="\r") 227 self._debug("Processing info...") 228 self._th_generate_xml_catalog(applist,outdir,info_url,repo_type,progress_bar) 229 self._debug("Fetched repo "+repo) 230 # print (("Removing old entries...")) 231 self._clean_bundle_catalogue(applist,outdir) 221 self._debug("Fetching repo %s"%repo_info['url']) 222 ## print (("Fetching %s catalogue: "+progress_bar)%repo_type,end="\r") 223 ## progress_bar=progress_bar+"#" 224 ## print (("Fetching %s catalogue: "+progress_bar)%repo_type,end="\r") 225 applist=self._generate_applist(self._fetch_repo(repo_info['url']),repo_name) 226 ## progress_bar=progress_bar+"##" 227 ## print (("Fetching %s catalogue: "+progress_bar)%repo_type,end="\r") 228 self._debug("Processing info...") 229 self._th_generate_xml_catalog(applist,outdir,repo_info['url_info'],repo_info['url'],repo_name,progress_bar) 230 self._debug("Fetched repo "+repo_info['url']) 231 all_apps.extend(applist) 232 232 else: 233 233 self._debug("appImage catalogue could not be fetched: Permission denied") 234 #Load external apps 235 for app_name,app_info in self.external_appimages.items(): 236 if self._chk_bundle_dir(outdir): 237 appimage=app_info['url'].split('/')[-1] 238 appimage_url='/'.join(app_info['url'].split('/')[0:-1]) 239 self._debug("Fetching external appimage %s"%app_info['url']) 240 applist=[appimage] 241 self._debug("Processing info...") 242 self._th_generate_xml_catalog(applist,outdir,app_info['url_info'],appimage_url,app_name,progress_bar) 243 self._debug("Fetched appimage "+app_info['url']) 244 all_apps.extend(applist) 245 else: 246 self._debug("External appImage could not be fetched: Permission denied") 247 248 249 ## print (("Removing old entries...")) 250 self._clean_bundle_catalogue(all_apps,outdir) 234 251 return(True) 235 252 #def _download_bundles_catalogue … … 247 264 248 265 def _fetch_repo(self,repo): 249 with urllib.request.urlopen(repo) as f: 266 req=Request(repo, headers={'User-Agent':'Mozilla/5.0'}) 267 with urllib.request.urlopen(req) as f: 250 268 content=(f.read().decode('utf-8')) 269 251 270 return(content) 252 271 #def _fetch_repo 253 272 254 def _generate_applist(self,content ):273 def _generate_applist(self,content,repo_name): 255 274 garbage_list=[] 256 275 applist=[] 257 garbage_list=content.split(' ') 258 for garbage_line in garbage_list: 259 if garbage_line.endswith('AppImage"'): 260 app=garbage_line.replace('href=":','') 261 applist.append(app.replace('"','')) 276 #webscrapping for probono repo 277 if repo_name=='probono': 278 garbage_list=content.split(' ') 279 for garbage_line in garbage_list: 280 if garbage_line.endswith('AppImage"'): 281 app=garbage_line.replace('href=":','') 282 applist.append(app.replace('"','')) 283 #Example of a webscrapping from another site 284 # if repo_name='other_repo_name': 285 # for garbage_line in garbage_list: 286 # if garbage_line.startswith('file="'): 287 # if 'appimage' in garbage_line: 288 # app=garbage_line.replace('file="','') 289 # app=app.replace('\n','') 290 # self._debug("Add %s"%app) 291 # applist.append(app.replace('"','')) 292 262 293 return(applist) 263 294 #def _generate_applist 264 295 265 def _th_generate_xml_catalog(self,applist,outdir,info_url,repo _type,progress_bar=''):296 def _th_generate_xml_catalog(self,applist,outdir,info_url,repo,repo_name,progress_bar=''): 266 297 CURSOR_UP='\033[F' 267 298 ERASE_LINE='\033[K' … … 274 305 # print (CURSOR_UP) 275 306 for app in random_applist: 276 th=threading.Thread(target=self._th_write_xml, args = (app,outdir,info_url, semaphore,inc))307 th=threading.Thread(target=self._th_write_xml, args = (app,outdir,info_url,repo,repo_name,semaphore,inc)) 277 308 th.start() 278 309 # os.system('setterm -cursor off') … … 285 316 #def _th_generate_xml_catalog 286 317 287 def _th_write_xml(self,app,outdir,info_url, semaphore,inc):318 def _th_write_xml(self,app,outdir,info_url,repo,repo_name,semaphore,inc): 288 319 semaphore.acquire() 289 320 lock=threading.Lock() … … 295 326 self._debug("checking if we need to download "+filename) 296 327 if not os.path.isfile(filename): 297 self._write_xml_file(filename,app,name,version,info_url, lock)328 self._write_xml_file(filename,app,name,version,info_url,repo,repo_name,lock) 298 329 with lock: 299 330 self.progress=self.progress+inc … … 301 332 #def _th_write_xml 302 333 303 def _write_xml_file(self,filename,app,name,version,info_url, lock):334 def _write_xml_file(self,filename,app,name,version,info_url,repo,repo_name,lock): 304 335 self._debug("Generating "+app+" xml") 305 336 f=open(filename,'w') … … 321 352 description=self.descriptions_dict[name] 322 353 else: 323 description=self._get_description(name,info_url )354 description=self._get_description(name,info_url,repo_name) 324 355 self.descriptions_dict.update({name:description}) 325 356 except: 326 357 description='' 327 358 summary=' '.join(list(description.split(' ')[:8])) 328 description="This is an AppImage bundle of app "+name+". It hasn't been tested by our developers and comes from a 3rd party dev team. Please use it carefully. "359 description="This is an AppImage bundle of app "+name+". It hasn't been tested by our developers and comes from a 3rd party dev team. Please use it carefully.\n"+description 329 360 if not summary: 330 361 summary=' '.join(list(description.split(' ')[:8])) 331 362 f.write(" <description><p></p><p>"+description+"</p></description>\n") 332 363 f.write(" <summary>"+summary+"...</summary>\n") 364 f.write(' <url type="homepage">'+repo+'</url>\n') 333 365 f.write(" <bundle type=\"appimage\">"+app+"</bundle>\n") 334 366 f.write(" <keywords>\n") … … 346 378 #def _write_xml_file 347 379 348 def _get_description(self,app_name,info_url ):380 def _get_description(self,app_name,info_url,repo_name): 349 381 desc='' 350 382 if '$(appname)' in info_url: 351 383 info_url=info_url.replace('$(appname)',app_name) 352 self._debug("Getting description from "+info_url) 353 try: 354 with urllib.request.urlopen(info_url) as f: 355 content=(f.read().decode('utf-8')) 356 soup=BeautifulSoup(content,"html.parser") 357 description_div=soup.findAll('div', attrs={ "class" : "description-text"}) 358 if len(description_div)>0: 359 desc=description_div[0].text 360 desc=desc.replace(':','.') 361 desc=desc.replace('&','&') 362 except Exception as e: 363 print("Can't get description from "+info_url) 364 print(str(e)) 365 pass 384 if info_url: 385 self._debug("Getting description from repo/app %s - %s "%(repo_name,info_url)) 386 try: 387 with urllib.request.urlopen(info_url) as f: 388 if repo_name=='probono': 389 content=(f.read().decode('utf-8')) 390 soup=BeautifulSoup(content,"html.parser") 391 description_div=soup.findAll('div', attrs={ "class" : "description-text"}) 392 if len(description_div)>0: 393 desc=description_div[0].text 394 desc=desc.replace(':','.') 395 desc=desc.replace('&','&') 396 except Exception as e: 397 print("Can't get description from "+info_url) 398 print(str(e)) 399 pass 366 400 return(desc) 367 401 #def _get_description -
lliurex-store/trunk/fuentes/python3-lliurex-store.install/usr/share/lliurexstore/plugins/snapManager.py
r6693 r7069 15 15 16 16 def __init__(self): 17 self.dbg= False17 self.dbg=True 18 18 self.progress=0 19 19 self.partial_progress=0 20 20 self.plugin_actions={'install':'snap','remove':'snap','pkginfo':'snap','load':'snap'} 21 self.cache_dir= "/var/lib/lliurexstore/bundles/cache"21 self.cache_dir=os.getenv("HOME")+"/.cache/lliurex-store" 22 22 self.icons_folder=self.cache_dir+"/icons" 23 self.images_folder=self.cache_dir+"/images" 23 24 self.result={} 24 25 self.result['data']={} … … 114 115 bundle=appstream.Bundle() 115 116 app=appstream.App() 117 icon=appstream.Icon() 118 screenshot=appstream.Screenshot() 116 119 #F*****g appstream have kinds undefined but kind_from_string works... wtf? 117 120 # bundle.set_kind(appstream.BundleKind.SNAP) … … 136 139 137 140 if pkg.get_icon(): 138 app.set_icon_path(pkg.get_icon()) 141 if self.icon_cache_enabled: 142 icon.set_kind(appstream.IconKind.LOCAL) 143 icon.set_name(self._download_file(pkg.get_icon(),pkg.get_name())) 144 else: 145 icon.set_kind(appstream.IconKind.REMOTE) 146 icon.set_name(pkg.get_icon()) 147 app.add_icon(icon) 148 149 if pkg.get_license(): 150 app.set_project_license(pkg.get_license()) 151 152 # if pkg.get_screenshots(): 153 # img=appstream.Image() 154 # screenshot_list=[] 155 # for screen in pkg.get_screenshots(): 156 # screenshot_list.append(screen.get_url()) 157 158 # app_info["screenshots"]=screenshot_list 139 159 return(app) 140 160 … … 167 187 self._debug("Searching %s"%tokens) 168 188 pkg=None 189 pkgs=None 169 190 try: 170 191 pkgs=self.snap_client.find_sync(Snapd.FindFlags.MATCH_NAME,tokens,None,None) … … 176 197 if pkg.get_channel()=='stable': 177 198 stable_pkgs.append(pkg) 199 else: 200 self._debug(pkg.get_channel()) 178 201 self._debug("Done") 179 202 return(stable_pkgs) … … 206 229 #switch to launch async method when running under a gui 207 230 #For an unknown reason request will block when sync mode under a gui and async blocks when on cli (really funny) 208 if self.cli_mode: 209 pkgs=self._search_snap(app_info['name']) 210 else: 211 pkgs=self._search_snap_async(app_info['name']) 231 self._debug("Getting info for %s"%app_info) 232 pkg=None 233 try: 234 pkg=self.snap_client.list_one_sync(app_info['name']) 235 app_info['state']='installed' 236 pkgs=[pkg] 237 except: 238 app_info['state']='available' 239 if self.cli_mode: 240 pkgs=self._search_snap(app_info['name']) 241 else: 242 pkgs=self._search_snap_async(app_info['name']) 243 self._debug("Getting extended info for %s %s"%(app_info['name'],pkgs)) 212 244 if type(pkgs)==type([]): 213 245 for pkg in pkgs: 246 self._debug("Getting extended info for %s"%app_info['name']) 214 247 if pkg.get_download_size(): 215 248 app_info['size']=str(pkg.get_download_size()) 216 249 else: 217 250 app_info['size']=str(pkg.get_installed_size()) 218 if pkg.get_icon():219 if self.icon_cache_enabled:220 app_info['icon']=self._download_file(pkg.get_icon(),app_info['name'])221 else:222 app_info['icon']=pkg.get_icon()223 if pkg.get_status():224 #Not working on xenial225 # if pkg.get_status()==Snapd.SnapStatus.INSTALLED or pkg.get_status()==3:226 # app_info['state']='installed'227 try:228 self.snap_client.list_one_sync(pkg.get_name())229 app_info['state']='installed'230 except:231 app_info['state']='available'232 if pkg.get_screenshots():233 screenshot_list=[]234 for screen in pkg.get_screenshots():235 screenshot_list.append(screen.get_url())236 app_info["screenshots"]=screenshot_list237 #Method not working in xenial, license default type assigned in infoManager238 # if pkg.get_license():239 # app_info["license"]=pkg.get_license()240 251 break 241 252 else:
Note: See TracChangeset
for help on using the changeset viewer.