py_artecs.tap

  1__DESCRIPTION__="""
  2
  3TAP services for ARTECS
  4
  5By M.Maris 1.1 - 2019 Nov 15 - 
  6Ported to python3
  7Needs pyvo 
  8
  9
 10
 11Example:
 12   >>> import artecs
 13   >>> atap=artecs.exop_pubblic_tap()
 14   
 15   >>> atap.EXPLAIN()
 16   
 17   >>> atap.keys()
 18
 19
 20   >>> tab=atap.search('(0.7 <= SMA) and (SMA <=3.)')
 21
 22   >>> tab.FO_CONST.unique()
 23
 24   >>> tab.to_csv('/tmp/pippo.csv',sep=' ')
 25
 26   >>> MAP=atap.get_map(tab.URL[0])
 27
 28"""
 29
 30try :
 31   import pyvo
 32   BAD=False
 33except :
 34   print("PYVO not found, to install pyvo : \n\t>sudo pip install pyvo\n%s EXOP_TAP class will not work properly"%("/".join(__file__.split('/')[-2:])))
 35   BAD=True
 36
 37#if BAD : 
 38   #import sys
 39   #sys.exit(1)
 40#del BAD
 41
 42class EXOP_TAP :
 43   def __init__(self,tap_url="http://archives.ia2.inaf.it/vo/tap/exo",table_name="exo.EXO",temporary_files_path='/tmp') :
 44      import pyvo
 45      #
 46      #creates empty object
 47      self._empty()
 48      #
 49      self._temporary_files_path=temporary_files_path+'/' if temporary_files_path!='' and temporary_files_path!=None else ''
 50      #
 51      #connects to db 
 52      self._url=tap_url
 53      self._table_name=table_name
 54      #
 55      #template for query_string, two fields: <QUERY> <FIELDS>
 56      self._template_query="SELECT %s %s FROM "+self._table_name
 57      #
 58      try :
 59         self._tap_service = pyvo.dal.TAPService(tap_url)
 60      except :
 61         print ("Error, unable to connect to TAP_URL '"+tap_url+"'")
 62      #
 63      #from db gets fields description
 64      self._get_field_description()
 65   #
 66   def _empty(self) :
 67      self._url=None
 68      self._table_name=None
 69      self._tap_service = None
 70      self._elapsed_time=0.
 71      self._search_result=None
 72      self._field_description=None
 73      self._field_names=None
 74      self._search_success=None
 75      self._template_query=None
 76      self._download_success=None
 77   #
 78   def _get_field_description(self) :
 79      self.adql_search('SELECT TOP 1 * FROM '+self._table_name)
 80      self._field_description=self._search_result.fielddescs
 81      self._field_names=self._search_result.fieldnames
 82      self.clean()
 83   #
 84   def keys(self) :
 85      """list of fields in the database"""
 86      return self._field_names
 87   #
 88   def has_key(self,this) :
 89      """has_key"""
 90      return this in self._field_names
 91   #
 92   def __include__(self,this) :
 93      """this in object"""
 94      return this in self._field_names
 95   #
 96   def success(self) : 
 97      """returns True if last search was successfull"""
 98      return self._search_success==True
 99   #
100   def clean(self) :
101      """cleans information from last search"""
102      self._search_success=None
103      self._search_result=None
104   #
105   def adql_search(self,adql_string) :
106      """search on database using ADQL string """
107      import time
108      self.clean()
109      self._search_success=False
110      self._elapsed_time=time.time()
111      self._search_result=self._tap_service.search(adql_string)
112      self._elapsed_time=time.time()-self._elapsed_time
113      self._search_success=True
114   #
115   def search_string(self,SELECTION,TOP,FIELDS,WHERE,SORT) :
116      """creates a properly formatted adql query_string"""
117      adql='SELECT' 
118      adql+='' if type(SELECTION)==type(None) else ' '+SELECT
119      adql+=' TOP %d'%TOP if type(TOP)==type(1) else '' 
120      adql+=' *' if type(FIELDS)==type(None) else ' '+FIELDS
121      adql+=' FROM '+self._table_name 
122      adql+='' if WHERE=='' or WHERE==None else ' WHERE '+WHERE
123      adql+='' if SORT=='' or SORT==None else ' SORT BY '+SORT
124      return adql
125   #
126   def search(self,WHERE,SELECTION=None,FIELDS=None,TOP=None,SORT=None,as_astropy=False,as_votable=False) :
127      """download a table from the database
128         search('') or search(WHERE='') returns all the data in the database
129         
130         the table can be returned as :
131            pandas dataframe (default)
132            astropy table (as_astropy = True)
133            votable (as_votable=True)
134      """
135      import time
136      #
137      adql=self.search_string(SELECTION,TOP,FIELDS,WHERE,SORT)
138      self.adql_search(adql)
139      #
140      if as_astropy :
141         return self._search_result.table
142      elif as_votable :
143         return self._search_result.votable
144      else :
145         return self._search_result.table.to_pandas()
146   #
147   def get_map(self,_URL,outfile=None) :
148      """gets a map """
149      import numpy as np
150      import time
151      from .artecs_map import artecs_map
152      try :
153         from urllib2 import urlopen
154      except :
155         from urllib.request import urlopen
156      import requests
157      import gzip
158      import os
159      try :
160         from StringIO import StringIO
161      except :
162         from io import StringIO
163      if type(_URL) == type("") :
164         URL=_URL+''
165      else :
166         URL=_URL.decode("utf-8")
167      #
168      if type(outfile) != type('') :
169         #if out file not specified creates a unique temporary name and reserves it
170         flag = True
171         while flag :
172            tmp=URL+' '+time.asctime()
173            tmp=tmp+' '+str(np.random.randint(1000000))
174            tmp=abs(hash(tmp))
175            tmp=hex( tmp )[2:]
176            _o=self._temporary_files_path+'artecs_download_file_'+tmp+'.fits'
177            flag=os.path.isfile(_o)
178            open(_o,'w').write('a') #this is to reserve the file name
179      else :
180         _o=outfile
181      #
182      tic=time.time()
183      request=requests.Request(URL)
184      #response = urlopen(request)
185      try :
186         response = urlopen(URL)
187      except :
188         print("URL %s not found"%URL)
189         if type(outfile) != type('') :
190            os.remove(_o)
191         self._download_success=False
192         return
193      #buf = StringIO(response.read())
194      buf = response
195      f = gzip.GzipFile(fileobj=buf)
196      data=f.read()
197      open(_o,'wb').write(data)
198      tic=time.time()-tic
199      #
200      amap=artecs_map(_o)
201      amap.url=URL
202      if type(outfile) != type('') :
203         os.remove(_o)
204      self._download_success=True
205      return amap
206   #
207   def download_map(self,URL,outfile=None,path=None) :
208      """download the fits file corresponding at a given URL
209      if outfile is not specified the fits file name from the URL is used
210      if path is not specified the current file is stored in the current path
211      """
212      _path='' if type(path) != type('') else path+'/'
213      if type(outfile)!=type('') :
214         _o=URL.split('/')[-1].split('.gz')[0]
215         _o=_path+_o
216      else :
217         _o=outfile
218      self.get_map(URL,outfile=_o)
219   # 
220   def EXPLAIN(self) :
221      """print a short introduction on the query language"""
222      print ("""
223ADQL = Astronomical Data Query Language
224
225A query is a string 
226- SELECTION set of data to be selected
227- HOW_MANY elements to select
228- WHICH_FIELDS fields to select
229- FROM which table
230- WHERE condition for selection is true
231
232Example:
233
2341. Download all the data
235   SELECT * FROM exo.EXO
236   
2372. Download only the first 10 elements
238   SELECT TOP 10 * FROM exo.EXO
239
2403. Download only the first 10 elements with SMA in the range 0.9 to 1.1
241   SELECT TOP 10 * FROM exo.EXO WHERE SMA BETWEEN 0.9 AND 1.1
242   
243   alternativelly
244
245   SELECT TOP 10 * FROM exo.EXO WHERE (0.9 <= SMA) AND (SMA <= 1.1)
246
2474. Download only the first 10 elements with SMA in the range 0.9 to 1.1 and CONTHAB>=0.5
248   SELECT TOP 10 * FROM exo.EXO WHERE SMA BETWEEN 0.9 AND 1.1 AND CONTHAB>=0.5
249
2505. Arithmetic calculations are allowed if needed for selection so to download the first 10 elements with SMA^1.5 > 0.87 
251   SELECT TOP 10 * FROM exo.EXO WHERE power(SMA,1.5)> 0.87 
252   
2536. returns just columns SMA and CONTHAB from previous example:
254   SELECT TOP 10 SMA,CONTHAB FROM exo.EXO WHERE power(SMA,1.5)> 0.87 
255   
256Note that the query is not sensitive to uppercase or lowercase.
257
258Tutorials: 
259   http://www.g-vo.org/tutorials/gaia-mock-tap.pdf
260   http://www.ivoa.net/documents/REC/ADQL/ADQL-20081030.pdf
261   http://www.ivoa.net/documents/
262
263""")
264
265      
class EXOP_TAP:
 43class EXOP_TAP :
 44   def __init__(self,tap_url="http://archives.ia2.inaf.it/vo/tap/exo",table_name="exo.EXO",temporary_files_path='/tmp') :
 45      import pyvo
 46      #
 47      #creates empty object
 48      self._empty()
 49      #
 50      self._temporary_files_path=temporary_files_path+'/' if temporary_files_path!='' and temporary_files_path!=None else ''
 51      #
 52      #connects to db 
 53      self._url=tap_url
 54      self._table_name=table_name
 55      #
 56      #template for query_string, two fields: <QUERY> <FIELDS>
 57      self._template_query="SELECT %s %s FROM "+self._table_name
 58      #
 59      try :
 60         self._tap_service = pyvo.dal.TAPService(tap_url)
 61      except :
 62         print ("Error, unable to connect to TAP_URL '"+tap_url+"'")
 63      #
 64      #from db gets fields description
 65      self._get_field_description()
 66   #
 67   def _empty(self) :
 68      self._url=None
 69      self._table_name=None
 70      self._tap_service = None
 71      self._elapsed_time=0.
 72      self._search_result=None
 73      self._field_description=None
 74      self._field_names=None
 75      self._search_success=None
 76      self._template_query=None
 77      self._download_success=None
 78   #
 79   def _get_field_description(self) :
 80      self.adql_search('SELECT TOP 1 * FROM '+self._table_name)
 81      self._field_description=self._search_result.fielddescs
 82      self._field_names=self._search_result.fieldnames
 83      self.clean()
 84   #
 85   def keys(self) :
 86      """list of fields in the database"""
 87      return self._field_names
 88   #
 89   def has_key(self,this) :
 90      """has_key"""
 91      return this in self._field_names
 92   #
 93   def __include__(self,this) :
 94      """this in object"""
 95      return this in self._field_names
 96   #
 97   def success(self) : 
 98      """returns True if last search was successfull"""
 99      return self._search_success==True
100   #
101   def clean(self) :
102      """cleans information from last search"""
103      self._search_success=None
104      self._search_result=None
105   #
106   def adql_search(self,adql_string) :
107      """search on database using ADQL string """
108      import time
109      self.clean()
110      self._search_success=False
111      self._elapsed_time=time.time()
112      self._search_result=self._tap_service.search(adql_string)
113      self._elapsed_time=time.time()-self._elapsed_time
114      self._search_success=True
115   #
116   def search_string(self,SELECTION,TOP,FIELDS,WHERE,SORT) :
117      """creates a properly formatted adql query_string"""
118      adql='SELECT' 
119      adql+='' if type(SELECTION)==type(None) else ' '+SELECT
120      adql+=' TOP %d'%TOP if type(TOP)==type(1) else '' 
121      adql+=' *' if type(FIELDS)==type(None) else ' '+FIELDS
122      adql+=' FROM '+self._table_name 
123      adql+='' if WHERE=='' or WHERE==None else ' WHERE '+WHERE
124      adql+='' if SORT=='' or SORT==None else ' SORT BY '+SORT
125      return adql
126   #
127   def search(self,WHERE,SELECTION=None,FIELDS=None,TOP=None,SORT=None,as_astropy=False,as_votable=False) :
128      """download a table from the database
129         search('') or search(WHERE='') returns all the data in the database
130         
131         the table can be returned as :
132            pandas dataframe (default)
133            astropy table (as_astropy = True)
134            votable (as_votable=True)
135      """
136      import time
137      #
138      adql=self.search_string(SELECTION,TOP,FIELDS,WHERE,SORT)
139      self.adql_search(adql)
140      #
141      if as_astropy :
142         return self._search_result.table
143      elif as_votable :
144         return self._search_result.votable
145      else :
146         return self._search_result.table.to_pandas()
147   #
148   def get_map(self,_URL,outfile=None) :
149      """gets a map """
150      import numpy as np
151      import time
152      from .artecs_map import artecs_map
153      try :
154         from urllib2 import urlopen
155      except :
156         from urllib.request import urlopen
157      import requests
158      import gzip
159      import os
160      try :
161         from StringIO import StringIO
162      except :
163         from io import StringIO
164      if type(_URL) == type("") :
165         URL=_URL+''
166      else :
167         URL=_URL.decode("utf-8")
168      #
169      if type(outfile) != type('') :
170         #if out file not specified creates a unique temporary name and reserves it
171         flag = True
172         while flag :
173            tmp=URL+' '+time.asctime()
174            tmp=tmp+' '+str(np.random.randint(1000000))
175            tmp=abs(hash(tmp))
176            tmp=hex( tmp )[2:]
177            _o=self._temporary_files_path+'artecs_download_file_'+tmp+'.fits'
178            flag=os.path.isfile(_o)
179            open(_o,'w').write('a') #this is to reserve the file name
180      else :
181         _o=outfile
182      #
183      tic=time.time()
184      request=requests.Request(URL)
185      #response = urlopen(request)
186      try :
187         response = urlopen(URL)
188      except :
189         print("URL %s not found"%URL)
190         if type(outfile) != type('') :
191            os.remove(_o)
192         self._download_success=False
193         return
194      #buf = StringIO(response.read())
195      buf = response
196      f = gzip.GzipFile(fileobj=buf)
197      data=f.read()
198      open(_o,'wb').write(data)
199      tic=time.time()-tic
200      #
201      amap=artecs_map(_o)
202      amap.url=URL
203      if type(outfile) != type('') :
204         os.remove(_o)
205      self._download_success=True
206      return amap
207   #
208   def download_map(self,URL,outfile=None,path=None) :
209      """download the fits file corresponding at a given URL
210      if outfile is not specified the fits file name from the URL is used
211      if path is not specified the current file is stored in the current path
212      """
213      _path='' if type(path) != type('') else path+'/'
214      if type(outfile)!=type('') :
215         _o=URL.split('/')[-1].split('.gz')[0]
216         _o=_path+_o
217      else :
218         _o=outfile
219      self.get_map(URL,outfile=_o)
220   # 
221   def EXPLAIN(self) :
222      """print a short introduction on the query language"""
223      print ("""
224ADQL = Astronomical Data Query Language
225
226A query is a string 
227- SELECTION set of data to be selected
228- HOW_MANY elements to select
229- WHICH_FIELDS fields to select
230- FROM which table
231- WHERE condition for selection is true
232
233Example:
234
2351. Download all the data
236   SELECT * FROM exo.EXO
237   
2382. Download only the first 10 elements
239   SELECT TOP 10 * FROM exo.EXO
240
2413. Download only the first 10 elements with SMA in the range 0.9 to 1.1
242   SELECT TOP 10 * FROM exo.EXO WHERE SMA BETWEEN 0.9 AND 1.1
243   
244   alternativelly
245
246   SELECT TOP 10 * FROM exo.EXO WHERE (0.9 <= SMA) AND (SMA <= 1.1)
247
2484. Download only the first 10 elements with SMA in the range 0.9 to 1.1 and CONTHAB>=0.5
249   SELECT TOP 10 * FROM exo.EXO WHERE SMA BETWEEN 0.9 AND 1.1 AND CONTHAB>=0.5
250
2515. Arithmetic calculations are allowed if needed for selection so to download the first 10 elements with SMA^1.5 > 0.87 
252   SELECT TOP 10 * FROM exo.EXO WHERE power(SMA,1.5)> 0.87 
253   
2546. returns just columns SMA and CONTHAB from previous example:
255   SELECT TOP 10 SMA,CONTHAB FROM exo.EXO WHERE power(SMA,1.5)> 0.87 
256   
257Note that the query is not sensitive to uppercase or lowercase.
258
259Tutorials: 
260   http://www.g-vo.org/tutorials/gaia-mock-tap.pdf
261   http://www.ivoa.net/documents/REC/ADQL/ADQL-20081030.pdf
262   http://www.ivoa.net/documents/
263
264""")
EXOP_TAP( tap_url='http://archives.ia2.inaf.it/vo/tap/exo', table_name='exo.EXO', temporary_files_path='/tmp')
44   def __init__(self,tap_url="http://archives.ia2.inaf.it/vo/tap/exo",table_name="exo.EXO",temporary_files_path='/tmp') :
45      import pyvo
46      #
47      #creates empty object
48      self._empty()
49      #
50      self._temporary_files_path=temporary_files_path+'/' if temporary_files_path!='' and temporary_files_path!=None else ''
51      #
52      #connects to db 
53      self._url=tap_url
54      self._table_name=table_name
55      #
56      #template for query_string, two fields: <QUERY> <FIELDS>
57      self._template_query="SELECT %s %s FROM "+self._table_name
58      #
59      try :
60         self._tap_service = pyvo.dal.TAPService(tap_url)
61      except :
62         print ("Error, unable to connect to TAP_URL '"+tap_url+"'")
63      #
64      #from db gets fields description
65      self._get_field_description()
def keys(self):
85   def keys(self) :
86      """list of fields in the database"""
87      return self._field_names

list of fields in the database

def has_key(self, this):
89   def has_key(self,this) :
90      """has_key"""
91      return this in self._field_names

has_key

def success(self):
97   def success(self) : 
98      """returns True if last search was successfull"""
99      return self._search_success==True

returns True if last search was successfull

def clean(self):
101   def clean(self) :
102      """cleans information from last search"""
103      self._search_success=None
104      self._search_result=None

cleans information from last search

def search_string(self, SELECTION, TOP, FIELDS, WHERE, SORT):
116   def search_string(self,SELECTION,TOP,FIELDS,WHERE,SORT) :
117      """creates a properly formatted adql query_string"""
118      adql='SELECT' 
119      adql+='' if type(SELECTION)==type(None) else ' '+SELECT
120      adql+=' TOP %d'%TOP if type(TOP)==type(1) else '' 
121      adql+=' *' if type(FIELDS)==type(None) else ' '+FIELDS
122      adql+=' FROM '+self._table_name 
123      adql+='' if WHERE=='' or WHERE==None else ' WHERE '+WHERE
124      adql+='' if SORT=='' or SORT==None else ' SORT BY '+SORT
125      return adql

creates a properly formatted adql query_string

def search( self, WHERE, SELECTION=None, FIELDS=None, TOP=None, SORT=None, as_astropy=False, as_votable=False):
127   def search(self,WHERE,SELECTION=None,FIELDS=None,TOP=None,SORT=None,as_astropy=False,as_votable=False) :
128      """download a table from the database
129         search('') or search(WHERE='') returns all the data in the database
130         
131         the table can be returned as :
132            pandas dataframe (default)
133            astropy table (as_astropy = True)
134            votable (as_votable=True)
135      """
136      import time
137      #
138      adql=self.search_string(SELECTION,TOP,FIELDS,WHERE,SORT)
139      self.adql_search(adql)
140      #
141      if as_astropy :
142         return self._search_result.table
143      elif as_votable :
144         return self._search_result.votable
145      else :
146         return self._search_result.table.to_pandas()

download a table from the database search('') or search(WHERE='') returns all the data in the database

the table can be returned as : pandas dataframe (default) astropy table (as_astropy = True) votable (as_votable=True)

def get_map(self, _URL, outfile=None):
148   def get_map(self,_URL,outfile=None) :
149      """gets a map """
150      import numpy as np
151      import time
152      from .artecs_map import artecs_map
153      try :
154         from urllib2 import urlopen
155      except :
156         from urllib.request import urlopen
157      import requests
158      import gzip
159      import os
160      try :
161         from StringIO import StringIO
162      except :
163         from io import StringIO
164      if type(_URL) == type("") :
165         URL=_URL+''
166      else :
167         URL=_URL.decode("utf-8")
168      #
169      if type(outfile) != type('') :
170         #if out file not specified creates a unique temporary name and reserves it
171         flag = True
172         while flag :
173            tmp=URL+' '+time.asctime()
174            tmp=tmp+' '+str(np.random.randint(1000000))
175            tmp=abs(hash(tmp))
176            tmp=hex( tmp )[2:]
177            _o=self._temporary_files_path+'artecs_download_file_'+tmp+'.fits'
178            flag=os.path.isfile(_o)
179            open(_o,'w').write('a') #this is to reserve the file name
180      else :
181         _o=outfile
182      #
183      tic=time.time()
184      request=requests.Request(URL)
185      #response = urlopen(request)
186      try :
187         response = urlopen(URL)
188      except :
189         print("URL %s not found"%URL)
190         if type(outfile) != type('') :
191            os.remove(_o)
192         self._download_success=False
193         return
194      #buf = StringIO(response.read())
195      buf = response
196      f = gzip.GzipFile(fileobj=buf)
197      data=f.read()
198      open(_o,'wb').write(data)
199      tic=time.time()-tic
200      #
201      amap=artecs_map(_o)
202      amap.url=URL
203      if type(outfile) != type('') :
204         os.remove(_o)
205      self._download_success=True
206      return amap

gets a map

def download_map(self, URL, outfile=None, path=None):
208   def download_map(self,URL,outfile=None,path=None) :
209      """download the fits file corresponding at a given URL
210      if outfile is not specified the fits file name from the URL is used
211      if path is not specified the current file is stored in the current path
212      """
213      _path='' if type(path) != type('') else path+'/'
214      if type(outfile)!=type('') :
215         _o=URL.split('/')[-1].split('.gz')[0]
216         _o=_path+_o
217      else :
218         _o=outfile
219      self.get_map(URL,outfile=_o)

download the fits file corresponding at a given URL if outfile is not specified the fits file name from the URL is used if path is not specified the current file is stored in the current path

def EXPLAIN(self):
221   def EXPLAIN(self) :
222      """print a short introduction on the query language"""
223      print ("""
224ADQL = Astronomical Data Query Language
225
226A query is a string 
227- SELECTION set of data to be selected
228- HOW_MANY elements to select
229- WHICH_FIELDS fields to select
230- FROM which table
231- WHERE condition for selection is true
232
233Example:
234
2351. Download all the data
236   SELECT * FROM exo.EXO
237   
2382. Download only the first 10 elements
239   SELECT TOP 10 * FROM exo.EXO
240
2413. Download only the first 10 elements with SMA in the range 0.9 to 1.1
242   SELECT TOP 10 * FROM exo.EXO WHERE SMA BETWEEN 0.9 AND 1.1
243   
244   alternativelly
245
246   SELECT TOP 10 * FROM exo.EXO WHERE (0.9 <= SMA) AND (SMA <= 1.1)
247
2484. Download only the first 10 elements with SMA in the range 0.9 to 1.1 and CONTHAB>=0.5
249   SELECT TOP 10 * FROM exo.EXO WHERE SMA BETWEEN 0.9 AND 1.1 AND CONTHAB>=0.5
250
2515. Arithmetic calculations are allowed if needed for selection so to download the first 10 elements with SMA^1.5 > 0.87 
252   SELECT TOP 10 * FROM exo.EXO WHERE power(SMA,1.5)> 0.87 
253   
2546. returns just columns SMA and CONTHAB from previous example:
255   SELECT TOP 10 SMA,CONTHAB FROM exo.EXO WHERE power(SMA,1.5)> 0.87 
256   
257Note that the query is not sensitive to uppercase or lowercase.
258
259Tutorials: 
260   http://www.g-vo.org/tutorials/gaia-mock-tap.pdf
261   http://www.ivoa.net/documents/REC/ADQL/ADQL-20081030.pdf
262   http://www.ivoa.net/documents/
263
264""")

print a short introduction on the query language