Changeset 6819
- Timestamp:
- Feb 14, 2018, 6:21:53 PM (3 years ago)
- Location:
- lliurex-analytics-server/trunk/fuentes
- Files:
-
- 1 added
- 6 edited
Legend:
- Unmodified
- Added
- Removed
-
lliurex-analytics-server/trunk/fuentes/debian/changelog
r6768 r6819 1 lliurex-analytics-server (0.2.4) xenial; urgency=medium1 lliurex-analytics-server (0.2.4) unreleased; urgency=medium 2 2 3 3 * Fix test suite … … 9 9 * New cache for visualization graphs minimizing overloading and DoS attacks 10 10 * Improved scheduler with clients sending 10+ results 11 * Support to provide blacklist 12 * Support to platform data 11 13 12 14 -- M.Angel Juan <m.angel.juan@gmail.com> Mon, 05 Feb 2018 13:56:27 +0100 -
lliurex-analytics-server/trunk/fuentes/debian/postinst
r5926 r6819 261 261 fi 262 262 263 need_update_1_5=$(mysql $rootuser $rootpass -s -N -Danalytics -e "describe tmp_clients;" |grep ncpu |wc -l 2>/dev/null || true) 264 if [ "$need_update_1_5" = "0" ]; 265 echo "Updating database to 1.5 format" 266 mysql $rootuser $rootpass < /usr/lib/analytics-server/update_allow_platformdata.sql || true 267 fi 268 263 269 264 270 # CONFIG APACHE -
lliurex-analytics-server/trunk/fuentes/lliurex-analytics-server/usr/lib/analytics-server/analytics/db.php
r6767 r6819 144 144 } 145 145 146 function send_data($user,$version,$sabor,$apps,$date=''){ 146 function send_data($user,$version,$sabor,$apps,$specs,$date=''){ 147 $spec_sql_names = ''; 148 $spec_sql_values = ''; 149 if ($specs != false){ 150 try{ 151 $arch = $specs['arch']; 152 $mem = $specs['mem']; 153 if (is_numeric($mem)){ 154 $mem=(int)$mem; 155 } 156 $vga = $specs['vga']; 157 $cpu = $specs['cpu']['model']; 158 $ncpu = $specs['cpu']['ncpus']; 159 if (is_numeric($ncpu)){ 160 $ncpu=(int)$ncpu; 161 } 162 $spec_sql_names = ',arch,mem,vga,cpu,ncpu'; 163 $spec_sql_values = ",'$arch',$mem,'$vga','$cpu',$ncpu"; 164 }catch(Exception $e){ 165 $spec_sql_names = ''; 166 $spec_sql_values = ''; 167 } 168 } 169 if ($date == ''){ 170 $sql="INSERT INTO tmp_clients(user,version,sabor,status $spec_sql_names) values ('$user','$version','$sabor',0 $spec_sql_values)"; 171 }else{ 172 $sql="INSERT INTO tmp_clients(user,version,sabor,status,date $spec_sql_names) values ('$user','$version','$sabor',0,'$date' $spec_sql_values)"; 173 } 174 $retry=1; 175 $done=false; 176 $cli_id=false; 177 while (! $done and $retry < 4){ 178 $res=$this->dbconn->query($sql); 179 if ($res){ 180 $cli_id=$this->dbconn->insert_id; 181 $done=true; 182 }else{ 183 $retry+=1; 184 sleep($retry); 185 } 186 } 187 if ($retry == 4 or $cli_id == false) 188 throw new Exception('Error sending client data: '.$this->dbconn->error); 189 190 $err_apps=false; 191 $err_exception=false; 192 if (count($apps) != 0){ 147 193 if ($date == ''){ 148 $sql="INSERT INTO tmp_clients(user,version,sabor,status) values ('$user','$version','$sabor',0)"; 149 }else{ 150 $sql="INSERT INTO tmp_clients(user,version,sabor,status,date) values ('$user','$version','$sabor',0,'$date')"; 151 } 152 $retry=1; 153 $done=false; 154 $cli_id=false; 155 while (! $done and $retry < 4){ 156 $res=$this->dbconn->query($sql); 157 if ($res){ 158 $cli_id=$this->dbconn->insert_id; 159 $done=true; 194 $sql="insert into tmp_packages(client,app,value) values"; 195 }else{ 196 $sql="insert into tmp_packages(client,app,value,date) values"; 197 } 198 $values=array(); 199 // Prevent DoS attack 200 $i = 1000; 201 foreach ($apps as $app => $value){ 202 // Max 1000 apps 203 if ( $i > 0 ){ 204 $i = $i - 1; 160 205 }else{ 161 $retry+=1; 162 sleep($retry); 163 } 164 } 165 if ($retry == 4 or $cli_id == false) 166 throw new Exception('Error sending client data: '.$this->dbconn->error); 167 $err_apps=false; 168 $err_exception=false; 169 if (count($apps) != 0){ 206 throw new Exception('*** DoS detected, aborting more processing on this request ***'); 207 } 208 209 if (trim($app) == '' or trim($value) == ''){ 210 $err_apps=true; 211 $err_exception=new Exception('Wrong application values'); 212 continue; 213 } 170 214 if ($date == ''){ 171 $ sql="insert into tmp_packages(client,app,value) values";215 $values[]="($cli_id,'$app',$value)"; 172 216 }else{ 173 $sql="insert into tmp_packages(client,app,value,date) values"; 174 } 175 $values=array(); 176 foreach ($apps as $app => $value){ 177 if (trim($app) == '' or trim($value) == ''){ 178 $err_apps=true; 179 $err_exception=new Exception('Wrong application values'); 180 continue; 217 $values[]="($cli_id,'$app',$value,'$date')"; 218 } 219 } 220 if (count($values) > 0){ 221 $sql.=implode(',',$values); 222 $done=false; 223 $retry=1; 224 while (! $done and $retry < 4){ 225 $res=$this->dbconn->query($sql); 226 if ($res){ 227 $done=true; 228 }else{ 229 $retry += 1; 230 sleep($retry); 181 231 } 182 if ($date == ''){ 183 $values[]="($cli_id,'$app',$value)"; 184 }else{ 185 $values[]="($cli_id,'$app',$value,'$date')"; 186 } 187 } 188 if (count($values) > 0){ 189 $sql.=implode(',',$values); 190 $done=false; 191 $retry=1; 192 while (! $done and $retry < 4){ 193 $res=$this->dbconn->query($sql); 194 if ($res){ 195 $done=true; 196 }else{ 197 $retry += 1; 198 sleep($retry); 199 } 200 } 201 if ($retry == 4 or ! $done){ 202 $err_apps=true; 203 $err_exception=new Exception('Error sending client app data: '.$this->dbconn->error.' QUERY='.$sql); 204 } 205 } 206 } 207 //End operations 208 $sql = "Update tmp_clients set status=1 where id = $cli_id and status=0"; 209 $retry=1; 210 $done=false; 211 while (! $done and $retry < 4){ 212 $res=$this->dbconn->query($sql); 213 if ($res){ 214 $done=true; 215 }else{ 216 $retry+=1; 217 sleep($retry); 218 } 219 } 220 if ($retry == 4 or $cli_id == false){ 221 throw new Exception('Error commiting client data: '.$this->dbconn->error); 222 } 223 if ($err_apps){ 224 throw $err_exception; 225 } 226 } 232 } 233 if ($retry == 4 or ! $done){ 234 $err_apps=true; 235 $err_exception=new Exception('Error sending client app data: '.$this->dbconn->error.' QUERY='.$sql); 236 } 237 } 238 } 239 //End operations 240 $sql = "Update tmp_clients set status=1 where id = $cli_id and status=0"; 241 $retry=1; 242 $done=false; 243 while (! $done and $retry < 4){ 244 $res=$this->dbconn->query($sql); 245 if ($res){ 246 $done=true; 247 }else{ 248 $retry+=1; 249 sleep($retry); 250 } 251 } 252 if ($retry == 4 or $cli_id == false){ 253 throw new Exception('Error commiting client data: '.$this->dbconn->error); 254 } 255 if ($err_apps){ 256 throw $err_exception; 257 } 258 } 227 259 228 260 private function load_alias(){ -
lliurex-analytics-server/trunk/fuentes/lliurex-analytics-server/usr/lib/analytics-server/analytics/functions.php
r6767 r6819 25 25 return 'NOK'; 26 26 } 27 if (isset($data['specs'])){ 28 $specs=$data['specs']; 29 }else{ 30 $specs=false; 31 } 27 32 if (isset($data['date'])){ 28 $db->send_data($uid,$version,$sabor,json_decode($data['stats'],true),$ data['date']);33 $db->send_data($uid,$version,$sabor,json_decode($data['stats'],true),$specs,$data['date']); 29 34 }else{ 30 $db->send_data($uid,$version,$sabor,json_decode($data['stats'],true) );35 $db->send_data($uid,$version,$sabor,json_decode($data['stats'],true),$specs); 31 36 } 32 37 }catch (Exception $e){ … … 67 72 }; 68 73 } 74 75 function call_getlist(){ 76 return function($request,$reponse,$service){ 77 $list = array(); 78 $the_file = 'blacklist.txt'; 79 if ( !is_file($the_file)) { 80 return json_encode($list); 81 } 82 try{ 83 $content = file_get_contents('blacklist.txt'); 84 foreach (explode("\n",$content) as $item){ 85 $item = trim($item); 86 if ( $item != '') { 87 $list[] = $item; 88 } 89 } 90 }catch(Exception $e){} 91 return json_encode($list); 92 }; 93 } 94 95 69 96 70 97 function call_show_stats(){ -
lliurex-analytics-server/trunk/fuentes/lliurex-analytics-server/usr/lib/analytics-server/analytics/reports.php
r6767 r6819 19 19 $klein->respond('GET','/SystemStats',call_get_system_stats()); 20 20 $klein->respond('GET','/ShowSystemStats',call_show_system_stats()); 21 $klein->respond('GET','/getlist',call_getlist()); 21 22 $klein->respond('POST','/notify',call_bd()); 22 23 -
lliurex-analytics-server/trunk/fuentes/lliurex-analytics-server/usr/sbin/analyticsd
r6812 r6819 334 334 def get_client(self,*args,**kwargs): 335 335 try: 336 query="SELECT id,date,user,version,sabor from tmp_clients where status=1 LIMIT {}".format(int(self.mon.select_window))336 query="SELECT id,date,user,version,sabor,arch,mem,vga,cpu,ncpu from tmp_clients where status=1 LIMIT {}".format(int(self.mon.select_window)) 337 337 self.execute(query=query) 338 338 ret =[] 339 339 if self.cur.rowcount > 0: 340 340 for i in range(self.cur.rowcount): 341 v_id,v_date,v_user,v_version,v_flavour =self.cur.fetchone()341 v_id,v_date,v_user,v_version,v_flavour,v_arch,v_mem,v_vga,v_cpu,v_ncpu=self.cur.fetchone() 342 342 version=self.reduce_version(v_version) 343 343 flavour=self.reduce_flavour(version,v_flavour) 344 344 uuid = self.gen_uuid(v_date.month,v_date.year,v_user,v_version,v_flavour) 345 ret.append({'uuid':uuid,'id':v_id,'date':v_date,'uid':v_user,'version':version,'flavour':flavour,'rversion':v_version,'rflavour':v_flavour}) 345 if not v_arch: 346 v_arch = 'NULL' 347 if not v_mem: 348 v_mem = 'NULL' 349 if not v_vga: 350 v_vga = 'NULL' 351 if not v_cpu: 352 v_cpu = 'NULL' 353 if not v_ncpu: 354 v_ncpu = 'NULL' 355 ret.append({'uuid':uuid,'id':v_id,'date':v_date,'uid':v_user,'version':version,'flavour':flavour,'rversion':v_version,'rflavour':v_flavour,'arch':v_arch,'mem':v_mem,'vga':v_vga,'cpu':v_cpu,'ncpu':v_ncpu}) 346 356 return ret 347 357 else: … … 387 397 values= [] 388 398 for cli in kwargs['client_list']: 389 values.append("({},'{}','{}','{}','{}','{}','{}' )".format(cli['uuid'],cli['date'].strftime('%Y-%m-%d'),cli['uid'],cli['rversion'],cli['rflavour'],cli['version'],cli['flavour']))390 query = "INSERT INTO Client_Versions(`uuid`,`date`,`Client_uid`,`string_release`,`string_flavour`,`Releases_name`,`Flavours_name` ) VALUES {} on duplicate key update uuid=uuid".format(','.join(map(str,values)))399 values.append("({},'{}','{}','{}','{}','{}','{}','{}',{},'{}','{}',{})".format(cli['uuid'],cli['date'].strftime('%Y-%m-%d'),cli['uid'],cli['rversion'],cli['rflavour'],cli['version'],cli['flavour'],cli['arch'],cli['mem'],cli['vga'],cli['cpu'],cli['ncpu'])) 400 query = "INSERT INTO Client_Versions(`uuid`,`date`,`Client_uid`,`string_release`,`string_flavour`,`Releases_name`,`Flavours_name`,`arch`,`mem`,`vga`,`cpu`,`ncpu`) VALUES {} on duplicate key update uuid=uuid".format(','.join(map(str,values))) 391 401 self.execute(query=query) 392 402 return True
Note: See TracChangeset
for help on using the changeset viewer.