Source code for lcc.db_tier.connectors.ogle_client

from astropy.coordinates.sky_coordinate import SkyCoord
from bs4 import BeautifulSoup
import re
import urllib
import urllib2
import warnings

from lcc.db_tier.base_query import LightCurvesDb
from lcc.entities.exceptions import QueryInputError
from lcc.entities.star import Star
import numpy as np


[docs]class OgleII(LightCurvesDb): ''' Connector to OGLEII. It is divided into two subdatabases - "phot" and "bvi". The first one contains light curves and metadata about coordinates, identifiers and V magnitude. The second one also contains information about V and I. Identifier of the stars in OgleII db are: field, starid and target. In case of cone search (if coordinates are provided), "nearest" key can be used. If it is True just nearest star to the target point is returned. Example: -------- que1 = {"ra": 5.549147 * 15, "dec": -70.55792, "delta": 5, "nearest": True} que2 = {"field":"LMC_SC1","starid":"152248","target":"lmc"} client = StarsProvider().getProvider( obtain_method="OgleII", obtain_params=[que1, que2]) stars = client.getStarsWithCurves() ''' ROOT = "http://ogledb.astrouw.edu.pl/~ogle/photdb" BVI_TARGETS = ["lmc", "smc", "bul"] PHOT_TARGETS = ["lmc", "smc", "bul", "car"] QUERY_TYPES = ["bvi", "phot"] MAX_TIMEOUT = 60 LC_META = {"xlabel": "hjd", "xlabel_unit": "days", "ylabel": "magnitude", "ylabel_unit": "mag", "color": "V", "origin": "OgleII"} COL_MAP = {"Field": "field", "StarID": "starid", "RA": "ra", "Decl": "dec", "V": "v_mag", "I": "i_mag", "B": "b_mag"} def __init__(self, queries): ''' Parameters ---------- queries : list, dict, iterable Query is list of dictionaries of query parameters or single dictionary. ''' if isinstance(queries, dict): queries = [queries] self.queries = self._parseQueries(queries)
[docs] def getStarsWithCurves(self): return self.getStars(lc=True)
[docs] def getStars(self, lc=False): stars = [] for query in self.queries: stars += self.postQuery(query, lc) if "ra" in query and "dec" in query and "delta" in query: stars = self.coneSearch(SkyCoord(float(query["ra"]), float(query["dec"]), unit="deg"), stars, float(query["delta"] / 3600.), nearest=query.get("nearest", False)) return stars
[docs] def postQuery(self, query, lc): PAGE_LEN = 1e10 valmin_ra, valmax_ra, valmin_dec, valmax_dec = self._getRanges(query.get("ra"), query.get( "dec"), query.get("delta")) if valmax_ra and valmax_ra: valmax_ra = valmax_ra / 15. valmin_ra = valmin_ra / 15. params = { "db_target": query.get("target"), "dbtyp": "dia2", "sort": "field", "use_field": "field" in query, "val_field": query.get("field"), "use_starid": "starid" in query, "val_starid": query.get("starid"), "disp_starcat": "off", "use_starcat": "off", "disp_ra": "on", "use_ra": valmin_ra != "", "valmin_ra": valmin_ra, "valmax_ra": valmax_ra, "disp_decl": "on", "use_decl": valmin_dec != "", "valmin_decl": valmin_dec, "valmax_decl": valmax_dec, "disp_imean": "on", "use_imean": "mag_i_min" in query, "valmin_imean": query.get("mag_i_min"), "valmax_imean": query.get("mag_i_max"), "disp_pgood": "off", "disp_bmean": "on", "disp_vmean": "on", "disp_imean": "on", "disp_imed": "off", "disp_bsig": "off", "disp_vsig": "off", "disp_isig": "off", "disp_imederr": "off", "disp_ndetect": "off", "disp_v_i": "off", "disp_b_v": "off", "sorting": "ASC", "pagelen": PAGE_LEN, } # Delete unneeded parameters to_del = [] for key, value in params.iteritems(): if not value or value == "off": to_del.append(key) # Url for query url = "%s/query.php?qtype=%s&first=1" % (self.ROOT, query.get("db")) [params.pop(x, None) for x in to_del] result = urllib2.urlopen( url, urllib.urlencode(params), timeout=100) return self._parseResult(result, lc=lc)
def _parseQueries(self, queries): todel_queries = [] new_queries = [] for i, query in enumerate(queries): if "db" not in query: query["db"] = self.QUERY_TYPES[0] if "coo" in query and isinstance(query["coo"], SkyCoord) and "delta" in query: todel_queries.append(i) coo = query["coo"] query["ra"] = coo.ra.degree query["dec"] = coo.dec.degree if "ra" in query and "dec" in query and "target" not in query: todel_queries.append(i) if query["db"] == "phot": targets = self.PHOT_TARGETS else: targets = self.BVI_TARGETS for target in targets: z = query.copy() z["target"] = target new_queries.append(z) elif "starid" in query: if "field" in query: query["target"] = query["field"][:3].lower() elif "field_num" in query and "target" in query: query["field"] = query[ "target"].upper() + "_SC" + str(query["field_num"]) else: raise QueryInputError("Unresolved target") if query["db"] not in self.QUERY_TYPES: raise QueryInputError( "Invalid db. Available OgleII databases: %s" % self.QUERY_TYPES) return [item for i, item in enumerate( queries) if i not in todel_queries] + new_queries def _parseResult(self, result, lc=False): START_TABLE = "<p><table" END_TABLE = "</table>" if result.code != 200: warnings.warn("Website has not returned 200 status") return [] lc_tmp = None raw_table = "" skip = True for line in result.readlines(): if skip: if line.strip().startswith(START_TABLE): raw_table += line[len("<p>"):] skip = False if lc and line.strip().startswith("<input type='hidden'"): tmpdir_pattern = re.compile( "<input type='hidden' name='tmpdir' value='(.*)'>") tmpdir = tmpdir_pattern.match(line) if tmpdir: lc_tmp = tmpdir.group(1) else: if line.strip().startswith(END_TABLE): break raw_table += line if not raw_table: return [] soup = BeautifulSoup(raw_table, "lxml") table = soup.find('table') rows = table.findAll('tr') res_rows = [] for tr in rows[1:]: cols = tr.findAll('td') res_cols = [] for td in cols: res_cols.append(td.find(text=True)) res_rows.append(res_cols) header = [th.find(text=True) for th in table.findAll("th")] return self._createStars(header, res_rows, lc_tmp) def _createStars(self, header, rows, lc_tmp): # [u'No', u'Field', u'StarID', u'RA', u'Decl', u'V', u'I', u'B'] # [u'No', u'Field', u'StarID', u'RA', u'Decl', u'I'] cols_map = self._parseHeader(header) stars = [] for row in rows: field = str(row[cols_map.get("field")]) starid = int(row[cols_map.get("starid")]) ra = float(row[cols_map.get("ra")]) dec = float(row[cols_map.get("dec")]) colors = ["i_mag", "b_mag", "v_mag"] more = {} for col in colors: if cols_map.get(col) and cols_map.get(col): try: more[col] = float(row[cols_map.get(col)]) except: pass name = field + "_" + str(starid) coo = (ra * 15, dec, "deg") ident = {"OgleII": {"name": name, "db_ident": {"field": field, "starid": starid}}} st = Star(ident, name, coo, more) if lc_tmp: lc = self._getLc(field, starid, lc_tmp) if lc and len(lc) != 0: st.putLightCurve(np.array(lc), meta=self.LC_META) stars.append(st) return stars def _parseHeader(self, header): cols_map = {} for i, col in enumerate(header): if col in self.COL_MAP.keys(): cols_map[self.COL_MAP[col]] = i return cols_map def _getLc(self, field, starid, lc_tmp): params = { "field": field, "starid": starid, "tmpdir": lc_tmp, "db": "DIA", "points": "good", } _url = "%s/getobj.php" % self.ROOT _result = urllib2.urlopen(_url, urllib.urlencode(params)) url = "%s/data/%s/%s_i_%s.dat" % (self.ROOT, lc_tmp, field.lower(), starid) result = urllib2.urlopen(url) if (result.code == 200): star_curve = [] for line in result.readlines(): parts = line.strip().split(" ") star_curve.append( [round(float(parts[0]), 4), round(float(parts[1]), 3), round(float(parts[2]), 3)]) return star_curve