py_artecs.tap
1__DESCRIPTION__=""" 2 3TAP services for ARTECS 4 5By M.Maris 1.1 - 2019 Nov 15 - 6Ported to python3 7Needs pyvo 8 9 10 11Example: 12 >>> import artecs 13 >>> atap=artecs.exop_pubblic_tap() 14 15 >>> atap.EXPLAIN() 16 17 >>> atap.keys() 18 19 20 >>> tab=atap.search('(0.7 <= SMA) and (SMA <=3.)') 21 22 >>> tab.FO_CONST.unique() 23 24 >>> tab.to_csv('/tmp/pippo.csv',sep=' ') 25 26 >>> MAP=atap.get_map(tab.URL[0]) 27 28""" 29 30try : 31 import pyvo 32 BAD=False 33except : 34 print("PYVO not found, to install pyvo : \n\t>sudo pip install pyvo\n%s EXOP_TAP class will not work properly"%("/".join(__file__.split('/')[-2:]))) 35 BAD=True 36 37#if BAD : 38 #import sys 39 #sys.exit(1) 40#del BAD 41 42class EXOP_TAP : 43 def __init__(self,tap_url="http://archives.ia2.inaf.it/vo/tap/exo",table_name="exo.EXO",temporary_files_path='/tmp') : 44 import pyvo 45 # 46 #creates empty object 47 self._empty() 48 # 49 self._temporary_files_path=temporary_files_path+'/' if temporary_files_path!='' and temporary_files_path!=None else '' 50 # 51 #connects to db 52 self._url=tap_url 53 self._table_name=table_name 54 # 55 #template for query_string, two fields: <QUERY> <FIELDS> 56 self._template_query="SELECT %s %s FROM "+self._table_name 57 # 58 try : 59 self._tap_service = pyvo.dal.TAPService(tap_url) 60 except : 61 print ("Error, unable to connect to TAP_URL '"+tap_url+"'") 62 # 63 #from db gets fields description 64 self._get_field_description() 65 # 66 def _empty(self) : 67 self._url=None 68 self._table_name=None 69 self._tap_service = None 70 self._elapsed_time=0. 71 self._search_result=None 72 self._field_description=None 73 self._field_names=None 74 self._search_success=None 75 self._template_query=None 76 self._download_success=None 77 # 78 def _get_field_description(self) : 79 self.adql_search('SELECT TOP 1 * FROM '+self._table_name) 80 self._field_description=self._search_result.fielddescs 81 self._field_names=self._search_result.fieldnames 82 self.clean() 83 # 84 def keys(self) : 85 """list of fields in the database""" 86 return self._field_names 87 # 88 def has_key(self,this) : 89 """has_key""" 90 return this in self._field_names 91 # 92 def __include__(self,this) : 93 """this in object""" 94 return this in self._field_names 95 # 96 def success(self) : 97 """returns True if last search was successfull""" 98 return self._search_success==True 99 # 100 def clean(self) : 101 """cleans information from last search""" 102 self._search_success=None 103 self._search_result=None 104 # 105 def adql_search(self,adql_string) : 106 """search on database using ADQL string """ 107 import time 108 self.clean() 109 self._search_success=False 110 self._elapsed_time=time.time() 111 self._search_result=self._tap_service.search(adql_string) 112 self._elapsed_time=time.time()-self._elapsed_time 113 self._search_success=True 114 # 115 def search_string(self,SELECTION,TOP,FIELDS,WHERE,SORT) : 116 """creates a properly formatted adql query_string""" 117 adql='SELECT' 118 adql+='' if type(SELECTION)==type(None) else ' '+SELECT 119 adql+=' TOP %d'%TOP if type(TOP)==type(1) else '' 120 adql+=' *' if type(FIELDS)==type(None) else ' '+FIELDS 121 adql+=' FROM '+self._table_name 122 adql+='' if WHERE=='' or WHERE==None else ' WHERE '+WHERE 123 adql+='' if SORT=='' or SORT==None else ' SORT BY '+SORT 124 return adql 125 # 126 def search(self,WHERE,SELECTION=None,FIELDS=None,TOP=None,SORT=None,as_astropy=False,as_votable=False) : 127 """download a table from the database 128 search('') or search(WHERE='') returns all the data in the database 129 130 the table can be returned as : 131 pandas dataframe (default) 132 astropy table (as_astropy = True) 133 votable (as_votable=True) 134 """ 135 import time 136 # 137 adql=self.search_string(SELECTION,TOP,FIELDS,WHERE,SORT) 138 self.adql_search(adql) 139 # 140 if as_astropy : 141 return self._search_result.table 142 elif as_votable : 143 return self._search_result.votable 144 else : 145 return self._search_result.table.to_pandas() 146 # 147 def get_map(self,_URL,outfile=None) : 148 """gets a map """ 149 import numpy as np 150 import time 151 from .artecs_map import artecs_map 152 try : 153 from urllib2 import urlopen 154 except : 155 from urllib.request import urlopen 156 import requests 157 import gzip 158 import os 159 try : 160 from StringIO import StringIO 161 except : 162 from io import StringIO 163 if type(_URL) == type("") : 164 URL=_URL+'' 165 else : 166 URL=_URL.decode("utf-8") 167 # 168 if type(outfile) != type('') : 169 #if out file not specified creates a unique temporary name and reserves it 170 flag = True 171 while flag : 172 tmp=URL+' '+time.asctime() 173 tmp=tmp+' '+str(np.random.randint(1000000)) 174 tmp=abs(hash(tmp)) 175 tmp=hex( tmp )[2:] 176 _o=self._temporary_files_path+'artecs_download_file_'+tmp+'.fits' 177 flag=os.path.isfile(_o) 178 open(_o,'w').write('a') #this is to reserve the file name 179 else : 180 _o=outfile 181 # 182 tic=time.time() 183 request=requests.Request(URL) 184 #response = urlopen(request) 185 try : 186 response = urlopen(URL) 187 except : 188 print("URL %s not found"%URL) 189 if type(outfile) != type('') : 190 os.remove(_o) 191 self._download_success=False 192 return 193 #buf = StringIO(response.read()) 194 buf = response 195 f = gzip.GzipFile(fileobj=buf) 196 data=f.read() 197 open(_o,'wb').write(data) 198 tic=time.time()-tic 199 # 200 amap=artecs_map(_o) 201 amap.url=URL 202 if type(outfile) != type('') : 203 os.remove(_o) 204 self._download_success=True 205 return amap 206 # 207 def download_map(self,URL,outfile=None,path=None) : 208 """download the fits file corresponding at a given URL 209 if outfile is not specified the fits file name from the URL is used 210 if path is not specified the current file is stored in the current path 211 """ 212 _path='' if type(path) != type('') else path+'/' 213 if type(outfile)!=type('') : 214 _o=URL.split('/')[-1].split('.gz')[0] 215 _o=_path+_o 216 else : 217 _o=outfile 218 self.get_map(URL,outfile=_o) 219 # 220 def EXPLAIN(self) : 221 """print a short introduction on the query language""" 222 print (""" 223ADQL = Astronomical Data Query Language 224 225A query is a string 226- SELECTION set of data to be selected 227- HOW_MANY elements to select 228- WHICH_FIELDS fields to select 229- FROM which table 230- WHERE condition for selection is true 231 232Example: 233 2341. Download all the data 235 SELECT * FROM exo.EXO 236 2372. Download only the first 10 elements 238 SELECT TOP 10 * FROM exo.EXO 239 2403. Download only the first 10 elements with SMA in the range 0.9 to 1.1 241 SELECT TOP 10 * FROM exo.EXO WHERE SMA BETWEEN 0.9 AND 1.1 242 243 alternativelly 244 245 SELECT TOP 10 * FROM exo.EXO WHERE (0.9 <= SMA) AND (SMA <= 1.1) 246 2474. Download only the first 10 elements with SMA in the range 0.9 to 1.1 and CONTHAB>=0.5 248 SELECT TOP 10 * FROM exo.EXO WHERE SMA BETWEEN 0.9 AND 1.1 AND CONTHAB>=0.5 249 2505. Arithmetic calculations are allowed if needed for selection so to download the first 10 elements with SMA^1.5 > 0.87 251 SELECT TOP 10 * FROM exo.EXO WHERE power(SMA,1.5)> 0.87 252 2536. returns just columns SMA and CONTHAB from previous example: 254 SELECT TOP 10 SMA,CONTHAB FROM exo.EXO WHERE power(SMA,1.5)> 0.87 255 256Note that the query is not sensitive to uppercase or lowercase. 257 258Tutorials: 259 http://www.g-vo.org/tutorials/gaia-mock-tap.pdf 260 http://www.ivoa.net/documents/REC/ADQL/ADQL-20081030.pdf 261 http://www.ivoa.net/documents/ 262 263""") 264 265
class
EXOP_TAP:
43class EXOP_TAP : 44 def __init__(self,tap_url="http://archives.ia2.inaf.it/vo/tap/exo",table_name="exo.EXO",temporary_files_path='/tmp') : 45 import pyvo 46 # 47 #creates empty object 48 self._empty() 49 # 50 self._temporary_files_path=temporary_files_path+'/' if temporary_files_path!='' and temporary_files_path!=None else '' 51 # 52 #connects to db 53 self._url=tap_url 54 self._table_name=table_name 55 # 56 #template for query_string, two fields: <QUERY> <FIELDS> 57 self._template_query="SELECT %s %s FROM "+self._table_name 58 # 59 try : 60 self._tap_service = pyvo.dal.TAPService(tap_url) 61 except : 62 print ("Error, unable to connect to TAP_URL '"+tap_url+"'") 63 # 64 #from db gets fields description 65 self._get_field_description() 66 # 67 def _empty(self) : 68 self._url=None 69 self._table_name=None 70 self._tap_service = None 71 self._elapsed_time=0. 72 self._search_result=None 73 self._field_description=None 74 self._field_names=None 75 self._search_success=None 76 self._template_query=None 77 self._download_success=None 78 # 79 def _get_field_description(self) : 80 self.adql_search('SELECT TOP 1 * FROM '+self._table_name) 81 self._field_description=self._search_result.fielddescs 82 self._field_names=self._search_result.fieldnames 83 self.clean() 84 # 85 def keys(self) : 86 """list of fields in the database""" 87 return self._field_names 88 # 89 def has_key(self,this) : 90 """has_key""" 91 return this in self._field_names 92 # 93 def __include__(self,this) : 94 """this in object""" 95 return this in self._field_names 96 # 97 def success(self) : 98 """returns True if last search was successfull""" 99 return self._search_success==True 100 # 101 def clean(self) : 102 """cleans information from last search""" 103 self._search_success=None 104 self._search_result=None 105 # 106 def adql_search(self,adql_string) : 107 """search on database using ADQL string """ 108 import time 109 self.clean() 110 self._search_success=False 111 self._elapsed_time=time.time() 112 self._search_result=self._tap_service.search(adql_string) 113 self._elapsed_time=time.time()-self._elapsed_time 114 self._search_success=True 115 # 116 def search_string(self,SELECTION,TOP,FIELDS,WHERE,SORT) : 117 """creates a properly formatted adql query_string""" 118 adql='SELECT' 119 adql+='' if type(SELECTION)==type(None) else ' '+SELECT 120 adql+=' TOP %d'%TOP if type(TOP)==type(1) else '' 121 adql+=' *' if type(FIELDS)==type(None) else ' '+FIELDS 122 adql+=' FROM '+self._table_name 123 adql+='' if WHERE=='' or WHERE==None else ' WHERE '+WHERE 124 adql+='' if SORT=='' or SORT==None else ' SORT BY '+SORT 125 return adql 126 # 127 def search(self,WHERE,SELECTION=None,FIELDS=None,TOP=None,SORT=None,as_astropy=False,as_votable=False) : 128 """download a table from the database 129 search('') or search(WHERE='') returns all the data in the database 130 131 the table can be returned as : 132 pandas dataframe (default) 133 astropy table (as_astropy = True) 134 votable (as_votable=True) 135 """ 136 import time 137 # 138 adql=self.search_string(SELECTION,TOP,FIELDS,WHERE,SORT) 139 self.adql_search(adql) 140 # 141 if as_astropy : 142 return self._search_result.table 143 elif as_votable : 144 return self._search_result.votable 145 else : 146 return self._search_result.table.to_pandas() 147 # 148 def get_map(self,_URL,outfile=None) : 149 """gets a map """ 150 import numpy as np 151 import time 152 from .artecs_map import artecs_map 153 try : 154 from urllib2 import urlopen 155 except : 156 from urllib.request import urlopen 157 import requests 158 import gzip 159 import os 160 try : 161 from StringIO import StringIO 162 except : 163 from io import StringIO 164 if type(_URL) == type("") : 165 URL=_URL+'' 166 else : 167 URL=_URL.decode("utf-8") 168 # 169 if type(outfile) != type('') : 170 #if out file not specified creates a unique temporary name and reserves it 171 flag = True 172 while flag : 173 tmp=URL+' '+time.asctime() 174 tmp=tmp+' '+str(np.random.randint(1000000)) 175 tmp=abs(hash(tmp)) 176 tmp=hex( tmp )[2:] 177 _o=self._temporary_files_path+'artecs_download_file_'+tmp+'.fits' 178 flag=os.path.isfile(_o) 179 open(_o,'w').write('a') #this is to reserve the file name 180 else : 181 _o=outfile 182 # 183 tic=time.time() 184 request=requests.Request(URL) 185 #response = urlopen(request) 186 try : 187 response = urlopen(URL) 188 except : 189 print("URL %s not found"%URL) 190 if type(outfile) != type('') : 191 os.remove(_o) 192 self._download_success=False 193 return 194 #buf = StringIO(response.read()) 195 buf = response 196 f = gzip.GzipFile(fileobj=buf) 197 data=f.read() 198 open(_o,'wb').write(data) 199 tic=time.time()-tic 200 # 201 amap=artecs_map(_o) 202 amap.url=URL 203 if type(outfile) != type('') : 204 os.remove(_o) 205 self._download_success=True 206 return amap 207 # 208 def download_map(self,URL,outfile=None,path=None) : 209 """download the fits file corresponding at a given URL 210 if outfile is not specified the fits file name from the URL is used 211 if path is not specified the current file is stored in the current path 212 """ 213 _path='' if type(path) != type('') else path+'/' 214 if type(outfile)!=type('') : 215 _o=URL.split('/')[-1].split('.gz')[0] 216 _o=_path+_o 217 else : 218 _o=outfile 219 self.get_map(URL,outfile=_o) 220 # 221 def EXPLAIN(self) : 222 """print a short introduction on the query language""" 223 print (""" 224ADQL = Astronomical Data Query Language 225 226A query is a string 227- SELECTION set of data to be selected 228- HOW_MANY elements to select 229- WHICH_FIELDS fields to select 230- FROM which table 231- WHERE condition for selection is true 232 233Example: 234 2351. Download all the data 236 SELECT * FROM exo.EXO 237 2382. Download only the first 10 elements 239 SELECT TOP 10 * FROM exo.EXO 240 2413. Download only the first 10 elements with SMA in the range 0.9 to 1.1 242 SELECT TOP 10 * FROM exo.EXO WHERE SMA BETWEEN 0.9 AND 1.1 243 244 alternativelly 245 246 SELECT TOP 10 * FROM exo.EXO WHERE (0.9 <= SMA) AND (SMA <= 1.1) 247 2484. Download only the first 10 elements with SMA in the range 0.9 to 1.1 and CONTHAB>=0.5 249 SELECT TOP 10 * FROM exo.EXO WHERE SMA BETWEEN 0.9 AND 1.1 AND CONTHAB>=0.5 250 2515. Arithmetic calculations are allowed if needed for selection so to download the first 10 elements with SMA^1.5 > 0.87 252 SELECT TOP 10 * FROM exo.EXO WHERE power(SMA,1.5)> 0.87 253 2546. returns just columns SMA and CONTHAB from previous example: 255 SELECT TOP 10 SMA,CONTHAB FROM exo.EXO WHERE power(SMA,1.5)> 0.87 256 257Note that the query is not sensitive to uppercase or lowercase. 258 259Tutorials: 260 http://www.g-vo.org/tutorials/gaia-mock-tap.pdf 261 http://www.ivoa.net/documents/REC/ADQL/ADQL-20081030.pdf 262 http://www.ivoa.net/documents/ 263 264""")
EXOP_TAP( tap_url='http://archives.ia2.inaf.it/vo/tap/exo', table_name='exo.EXO', temporary_files_path='/tmp')
44 def __init__(self,tap_url="http://archives.ia2.inaf.it/vo/tap/exo",table_name="exo.EXO",temporary_files_path='/tmp') : 45 import pyvo 46 # 47 #creates empty object 48 self._empty() 49 # 50 self._temporary_files_path=temporary_files_path+'/' if temporary_files_path!='' and temporary_files_path!=None else '' 51 # 52 #connects to db 53 self._url=tap_url 54 self._table_name=table_name 55 # 56 #template for query_string, two fields: <QUERY> <FIELDS> 57 self._template_query="SELECT %s %s FROM "+self._table_name 58 # 59 try : 60 self._tap_service = pyvo.dal.TAPService(tap_url) 61 except : 62 print ("Error, unable to connect to TAP_URL '"+tap_url+"'") 63 # 64 #from db gets fields description 65 self._get_field_description()
def
success(self):
97 def success(self) : 98 """returns True if last search was successfull""" 99 return self._search_success==True
returns True if last search was successfull
def
clean(self):
101 def clean(self) : 102 """cleans information from last search""" 103 self._search_success=None 104 self._search_result=None
cleans information from last search
def
adql_search(self, adql_string):
106 def adql_search(self,adql_string) : 107 """search on database using ADQL string """ 108 import time 109 self.clean() 110 self._search_success=False 111 self._elapsed_time=time.time() 112 self._search_result=self._tap_service.search(adql_string) 113 self._elapsed_time=time.time()-self._elapsed_time 114 self._search_success=True
search on database using ADQL string
def
search_string(self, SELECTION, TOP, FIELDS, WHERE, SORT):
116 def search_string(self,SELECTION,TOP,FIELDS,WHERE,SORT) : 117 """creates a properly formatted adql query_string""" 118 adql='SELECT' 119 adql+='' if type(SELECTION)==type(None) else ' '+SELECT 120 adql+=' TOP %d'%TOP if type(TOP)==type(1) else '' 121 adql+=' *' if type(FIELDS)==type(None) else ' '+FIELDS 122 adql+=' FROM '+self._table_name 123 adql+='' if WHERE=='' or WHERE==None else ' WHERE '+WHERE 124 adql+='' if SORT=='' or SORT==None else ' SORT BY '+SORT 125 return adql
creates a properly formatted adql query_string
def
search( self, WHERE, SELECTION=None, FIELDS=None, TOP=None, SORT=None, as_astropy=False, as_votable=False):
127 def search(self,WHERE,SELECTION=None,FIELDS=None,TOP=None,SORT=None,as_astropy=False,as_votable=False) : 128 """download a table from the database 129 search('') or search(WHERE='') returns all the data in the database 130 131 the table can be returned as : 132 pandas dataframe (default) 133 astropy table (as_astropy = True) 134 votable (as_votable=True) 135 """ 136 import time 137 # 138 adql=self.search_string(SELECTION,TOP,FIELDS,WHERE,SORT) 139 self.adql_search(adql) 140 # 141 if as_astropy : 142 return self._search_result.table 143 elif as_votable : 144 return self._search_result.votable 145 else : 146 return self._search_result.table.to_pandas()
download a table from the database search('') or search(WHERE='') returns all the data in the database
the table can be returned as : pandas dataframe (default) astropy table (as_astropy = True) votable (as_votable=True)
def
get_map(self, _URL, outfile=None):
148 def get_map(self,_URL,outfile=None) : 149 """gets a map """ 150 import numpy as np 151 import time 152 from .artecs_map import artecs_map 153 try : 154 from urllib2 import urlopen 155 except : 156 from urllib.request import urlopen 157 import requests 158 import gzip 159 import os 160 try : 161 from StringIO import StringIO 162 except : 163 from io import StringIO 164 if type(_URL) == type("") : 165 URL=_URL+'' 166 else : 167 URL=_URL.decode("utf-8") 168 # 169 if type(outfile) != type('') : 170 #if out file not specified creates a unique temporary name and reserves it 171 flag = True 172 while flag : 173 tmp=URL+' '+time.asctime() 174 tmp=tmp+' '+str(np.random.randint(1000000)) 175 tmp=abs(hash(tmp)) 176 tmp=hex( tmp )[2:] 177 _o=self._temporary_files_path+'artecs_download_file_'+tmp+'.fits' 178 flag=os.path.isfile(_o) 179 open(_o,'w').write('a') #this is to reserve the file name 180 else : 181 _o=outfile 182 # 183 tic=time.time() 184 request=requests.Request(URL) 185 #response = urlopen(request) 186 try : 187 response = urlopen(URL) 188 except : 189 print("URL %s not found"%URL) 190 if type(outfile) != type('') : 191 os.remove(_o) 192 self._download_success=False 193 return 194 #buf = StringIO(response.read()) 195 buf = response 196 f = gzip.GzipFile(fileobj=buf) 197 data=f.read() 198 open(_o,'wb').write(data) 199 tic=time.time()-tic 200 # 201 amap=artecs_map(_o) 202 amap.url=URL 203 if type(outfile) != type('') : 204 os.remove(_o) 205 self._download_success=True 206 return amap
gets a map
def
download_map(self, URL, outfile=None, path=None):
208 def download_map(self,URL,outfile=None,path=None) : 209 """download the fits file corresponding at a given URL 210 if outfile is not specified the fits file name from the URL is used 211 if path is not specified the current file is stored in the current path 212 """ 213 _path='' if type(path) != type('') else path+'/' 214 if type(outfile)!=type('') : 215 _o=URL.split('/')[-1].split('.gz')[0] 216 _o=_path+_o 217 else : 218 _o=outfile 219 self.get_map(URL,outfile=_o)
download the fits file corresponding at a given URL if outfile is not specified the fits file name from the URL is used if path is not specified the current file is stored in the current path
def
EXPLAIN(self):
221 def EXPLAIN(self) : 222 """print a short introduction on the query language""" 223 print (""" 224ADQL = Astronomical Data Query Language 225 226A query is a string 227- SELECTION set of data to be selected 228- HOW_MANY elements to select 229- WHICH_FIELDS fields to select 230- FROM which table 231- WHERE condition for selection is true 232 233Example: 234 2351. Download all the data 236 SELECT * FROM exo.EXO 237 2382. Download only the first 10 elements 239 SELECT TOP 10 * FROM exo.EXO 240 2413. Download only the first 10 elements with SMA in the range 0.9 to 1.1 242 SELECT TOP 10 * FROM exo.EXO WHERE SMA BETWEEN 0.9 AND 1.1 243 244 alternativelly 245 246 SELECT TOP 10 * FROM exo.EXO WHERE (0.9 <= SMA) AND (SMA <= 1.1) 247 2484. Download only the first 10 elements with SMA in the range 0.9 to 1.1 and CONTHAB>=0.5 249 SELECT TOP 10 * FROM exo.EXO WHERE SMA BETWEEN 0.9 AND 1.1 AND CONTHAB>=0.5 250 2515. Arithmetic calculations are allowed if needed for selection so to download the first 10 elements with SMA^1.5 > 0.87 252 SELECT TOP 10 * FROM exo.EXO WHERE power(SMA,1.5)> 0.87 253 2546. returns just columns SMA and CONTHAB from previous example: 255 SELECT TOP 10 SMA,CONTHAB FROM exo.EXO WHERE power(SMA,1.5)> 0.87 256 257Note that the query is not sensitive to uppercase or lowercase. 258 259Tutorials: 260 http://www.g-vo.org/tutorials/gaia-mock-tap.pdf 261 http://www.ivoa.net/documents/REC/ADQL/ADQL-20081030.pdf 262 http://www.ivoa.net/documents/ 263 264""")
print a short introduction on the query language