Updating github to be the most up to date and working / development version.

Here, adding the wrapper_type functionality to fetch data in wrappers that return tables. Data class gains several new general wrapper classes that provide this new functionality.

Author: skdaccess

skdaccess/astro/kepler/data_fetcher.py

@@ -27,7 +27,7 @@
 """
 
 # mithagi required Base,Utils imports
-from skdaccess.framework.data_class import DataFetcherBase
+from skdaccess.framework.data_class import DataFetcherBase, DictionaryWrapper
 from skdaccess.utilities import trend_util
 from skdaccess.astro.kepler.data_wrapper import DataWrapper
 from skdaccess.utilities import data_util
@@ -46,7 +46,7 @@
 
 class DataFetcher(DataFetcherBase):
     ''' Data Fetcher for Kepler light curve data '''
-    def __init__(self, ap_paramList, normalize=False, drop_on_quality=False, filter_window=None, quarter_list=None):
+    def __init__(self, ap_paramList, normalize=False, drop_on_quality=False, filter_window=None, quarter_list=None, wrapper_type = 'series'):
         '''
         Initialize Kepler Data Fetcher
 
@@ -55,12 +55,14 @@ def __init__(self, ap_paramList, normalize=False, drop_on_quality=False, filter_
         @param drop_on_quality: Drop data if SAP_QUALITY != 0 
         @param filter_window: Size of window used when removing systematic offsets.
         @param quarter_list: List of quarters (0-17)
+        @param wrapper_type: Select the type of iterator wrapper to generate, as series or table
         '''
 
         self.normalize = normalize
         self.drop_on_quality = drop_on_quality
         self.filter_window = filter_window
         self.quarter_list = quarter_list
+        self.wrapper_type = wrapper_type
         super(DataFetcher, self).__init__(ap_paramList)
 
     def output(self):
@@ -125,5 +127,10 @@ def output(self):
                     data = kid_data[kid][kid_data[kid]['QUARTER'] == quarter].loc[:,'PDCSAP_FLUX']                    
                     kid_data[kid].update(data - trend_util.medianFilter(data, self.filter_window, interpolate=False))
 
-
-        return DataWrapper(kid_data)
+        if self.wrapper_type == 'series':
+            return DataWrapper(kid_data)
+        elif self.wrapper_type == 'table':
+            return DictionaryWrapper(kid_data)
+        else:
+            print('... Invald wrapper type, defaulting to series ...')
+            return DataWrapper(kid_data)

skdaccess/framework/data_class.py

@@ -70,7 +70,7 @@ def getMetadata(self):
 class DataWrapperBase(object):
     ''' Base class for wrapping data for use in DiscoveryPipeline. '''
 
-    def __init__(self, obj_wrap, run_id = -1):
+    def __init__(self, obj_wrap, run_id = -1, meta_data = None):
         '''
         Construct object from input data. 
 
@@ -81,6 +81,7 @@ def __init__(self, obj_wrap, run_id = -1):
         self.results = dict()
         self.constants = dict()
         self.run_id = run_id
+        self.meta_data = meta_data
 
     def update(self, obj):
         ''' Updated wrapped data '''
@@ -118,3 +119,60 @@ def getIterator(self):
     def getIndices(self):
         ''' Get indices of the data '''
         pass
+
+    def getLength(self):
+        pass
+
+class TableFetcher(DataFetcherBase):
+
+    def __init__(self, pd_data):
+
+        super(TableFetcher, self).__init__([])
+        self.pd_data = pd_data
+
+
+    def output(self):
+        return DataPanelWrapper(self.pd_data)
+
+
+class DataPanelWrapper(DataWrapperBase):
+    '''
+    Generic Data wrapper for pandas data panels. Iterators over data frames
+    '''
+    
+    def getIterator(self):
+        '''
+        Iterator access to data. Iterates over the minor axis.
+
+        @return iterator to data frames from Panel
+        '''
+        for label in self.data.minor_axis:
+            yield label, self.data[:,:,label]
+
+    def getLength(self):
+        '''
+        @return Number of data frames
+        '''
+        return len(self.data.minor_axis)
+            
+
+class DictionaryWrapper(DataWrapperBase):
+    '''
+    Generic data wrapper for a dictionary of data frames
+    '''
+    
+    def getIterator(self):
+        '''
+        Iterator access to data. Iterates over the minor axis.
+
+        @return iterator to data frames from Panel
+        '''        
+        for label in self.data.keys():
+            yield label, self.data[label]
+
+
+    def getLength(self):
+        '''
+        @return Number of data frames
+        '''
+        return len(self.data)

skdaccess/geo/grace/data_fetcher.py

@@ -27,7 +27,7 @@
 """
 
 # mithagi required Base imports
-from skdaccess.framework.data_class import DataFetcherBase
+from skdaccess.framework.data_class import DataFetcherBase, DataPanelWrapper
 from skdaccess.geo.grace.data_wrapper import DataWrapper
 from skdaccess.utilities.data_util import getDataLocation
 
@@ -38,19 +38,21 @@
 class DataFetcher(DataFetcherBase):
     ''' Data Fetcher for GRACE data '''
 
-    def __init__(self, ap_paramList, start_date = None, end_date = None, resample = True):
+    def __init__(self, ap_paramList, start_date = None, end_date = None, resample = True, wrapper_type='series'):
         '''
         Construct a Grace Data Fetcher
 
         @param ap_paramList[geo_pont]: Geographic location of grace data to select
         @param start_date: Beginning date
         @param end_date: Ending date
         @param resample: Resample the data to daily resolution, leaving NaN's in days without data (Default True)
+        @param wrapper_type: Select the type of iterator wrapper to generate, as series or table
         '''
 
         self.start_date = start_date
         self.end_date = end_date
         self.resample = resample
+        self.wrapper_type = wrapper_type
         super(DataFetcher, self).__init__(ap_paramList)
 
     def output(self):
@@ -106,7 +108,15 @@ def output(self):
         if self.resample == True:
             grace_data = grace_data.reindex(pd.date_range(start_date, end_date))
 
-        return(DataWrapper(grace_data))
+        if self.wrapper_type == 'series':
+            return(DataWrapper(grace_data))
+        elif self.wrapper_type == 'table':
+            grace_data.columns = ['Equivalent Water Thickness (cm)', 'Uncertainty']
+            return(DataPanelWrapper(pd.Panel.from_dict({'GRACE':grace_data},orient='minor')))
+        else:
+            print('... Invald wrapper type, defaulting to series ...')
+            return(DataWrapper(grace_data))
+            
 
     def __str__(self):
         '''

skdaccess/geo/groundwater/data_fetcher.py

@@ -27,7 +27,7 @@
 """
 
 # mithagi required Base imports
-from skdaccess.framework.data_class import DataFetcherBase
+from skdaccess.framework.data_class import DataFetcherBase, DataPanelWrapper
 from skdaccess.utilities.data_util import getDataLocation
 from skdaccess.geo.groundwater.data_wrapper import DataWrapper
 
@@ -40,13 +40,16 @@ class DataFetcher(DataFetcherBase):
     '''
     Generates Data Wrappers of groundwater measurements taken in California
     '''
-    def __init__(self,  ap_paramList = [], start_date = None, end_date = None, cutoff=0.75, adjust_heights = False):
+    def __init__(self,  ap_paramList = [], start_date = None, end_date = None, cutoff=0.75, adjust_heights = False, wrapper_type='series'):
         ''' 
         Construct a Ground Water Data Fetcher
 
         @param ap_paramList[station_list]: List of stations (Optional)
         @param start_date: Starting date (defualt: None)
         @param end_date: Ending date (default: None)
+        @param cutoff: Specified required amount of data at a station
+        @param adjust_heights: Flag to indicate going to water height, instead of original depth to water
+        @param wrapper_type: Select the type of iterator wrapper to generate, as series or table
         '''
         self.meta_data = DataFetcher.getStationMetadata()
 
@@ -55,6 +58,7 @@ def __init__(self,  ap_paramList = [], start_date = None, end_date = None, cutof
         self.ap_paramList = ap_paramList
         self.cutoff = cutoff
         self.adjust_heights = adjust_heights
+        self.wrapper_type = wrapper_type
 
     def output(self):
         ''' 
@@ -102,8 +106,15 @@ def output(self):
 
         store.close()
 
-        data = DataWrapper(pd.Panel.from_dict(data_dict,orient='minor'), self.meta_data)
-        return data
+        if self.wrapper_type == 'series':
+            data = DataWrapper(pd.Panel.from_dict(data_dict,orient='minor'), self.meta_data)
+            return data
+        elif self.wrapper_type == 'table':
+            return(DataPanelWrapper(pd.Panel.from_dict(data_dict,orient='minor'), meta_data=self.meta_data))
+        else:
+            print('... Invald wrapper type, defaulting to series ...')
+            data = DataWrapper(pd.Panel.from_dict(data_dict,orient='minor'), self.meta_data)
+            
 
     def __str__(self):
         '''

skdaccess/geo/pbo/data_fetcher.py

@@ -27,7 +27,7 @@
 """
 
 # mithagi required Base,Utils imports
-from skdaccess.framework.data_class import DataFetcherBase
+from skdaccess.framework.data_class import DataFetcherBase, DataPanelWrapper
 from skdaccess.utilities import pbo_util
 from skdaccess.utilities import data_util
 
@@ -37,7 +37,6 @@
 # 3rd party package imports
 import pandas as pd
 import numpy as np
-import pickle
 
 
 
@@ -48,7 +47,7 @@ class DataFetcher(DataFetcherBase):
         2) stabilization area (for running stabilization)
     '''
 
-    def __init__(self, start_time, end_time, lat_range, lon_range, ap_paramList, mdyratio=.5, epFlag=1, stabFlag=1):
+    def __init__(self, start_time, end_time, lat_range, lon_range, ap_paramList, mdyratio=.5, ep_flag=1, stab_flag=1, wrapper_type='series'):
         ''' 
         Initialize a DataFetcher
 
@@ -58,21 +57,25 @@ def __init__(self, start_time, end_time, lat_range, lon_range, ap_paramList, mdy
         @param lon_range: Longitude range used to select stabilization sites
         @range ap_paramList[radius]: Site radius to search around (km)
         @param ap_paramList[geo_point] Tuple containing lat and lon coordinates
+        @param ep_flag: Propagate errors through stabilization
+        @param stab_flag: Perform regional stabilization
+        @param wrapper_type: Select the type of iterator wrapper to generate, as series or table
         '''
 
 
-        self.start_time = start_time
-        self.end_time = end_time
+        self._start_time = start_time
+        self._end_time = end_time
         self.ap_paramList = ap_paramList
-        self.geospace = (lat_range, lon_range)
+        self._geospace = (lat_range, lon_range)
         self.station_list = None
-        self.mdyratio = mdyratio
-        self.epFlag = epFlag
-        self.stabFlag = stabFlag
+        self._mdyratio = mdyratio
+        self._ep_flag = ep_flag
+        self._stab_flag = stab_flag
+        self.wrapper_type = wrapper_type
 
         self.meta_data = DataFetcher.getStationMetadata()
 
-        if stabFlag == 1:
+        if stab_flag == 1:
             self.stabilize()
         else:
             self.rawData()
@@ -119,17 +122,17 @@ def stabilize(self):
 
         storeData = pd.HDFStore(storeData_fn)
 
-        mdyratio = self.mdyratio
-        epFlag = self.epFlag
+        mdyratio = self._mdyratio
+        epFlag = self._ep_flag
 
         keyList =[]
         for ii in storeName.keys():
             coord = storeName[ii]['refNEU']
-            if coord[0]>self.geospace[0][0] and coord[0]<self.geospace[0][1] and coord[1]>self.geospace[1][0] and coord[1]<self.geospace[1][1]:
+            if coord[0]>self._geospace[0][0] and coord[0]<self._geospace[0][1] and coord[1]>self._geospace[1][0] and coord[1]<self._geospace[1][1]:
                 keyList.append(storeName[ii]['4ID'])
-            elif coord[0]>self.geospace[0][0] and coord[0]<self.geospace[0][1] and coord[1]>(360+self.geospace[1][0]) and coord[1]<(360+self.geospace[1][1]):
+            elif coord[0]>self._geospace[0][0] and coord[0]<self._geospace[0][1] and coord[1]>(360+self._geospace[1][0]) and coord[1]<(360+self._geospace[1][1]):
                 keyList.append(storeName[ii]['4ID'])
-        smSet_all, smHdr_all = pbo_util.stab_sys(storeName,storeData,[self.start_time,self.end_time],indx=keyList,mdyratio=mdyratio,errProp=epFlag)
+        smSet_all, smHdr_all = pbo_util.stab_sys(storeName,storeData,[self._start_time,self._end_time],indx=keyList,mdyratio=mdyratio,errProp=epFlag)
 
         self.smSet_all = smSet_all
         self.smHdr_all = smHdr_all
@@ -150,16 +153,16 @@ def rawData(self):
 
         storeData = pd.HDFStore(storeData_fn)
 
-        mdyratio = self.mdyratio
+        mdyratio = self._mdyratio
 
         keyList =[]
         for ii in storeName.keys():
             coord = storeName[ii]['refNEU']
-            if coord[0]>self.geospace[0][0] and coord[0]<self.geospace[0][1] and coord[1]>self.geospace[1][0] and coord[1]<self.geospace[1][1]:
+            if coord[0]>self._geospace[0][0] and coord[0]<self._geospace[0][1] and coord[1]>self._geospace[1][0] and coord[1]<self._geospace[1][1]:
                 keyList.append(storeName[ii]['4ID'])
-            elif coord[0]>self.geospace[0][0] and coord[0]<self.geospace[0][1] and coord[1]>(360+self.geospace[1][0]) and coord[1]<(360+self.geospace[1][1]):
+            elif coord[0]>self._geospace[0][0] and coord[0]<self._geospace[0][1] and coord[1]>(360+self._geospace[1][0]) and coord[1]<(360+self._geospace[1][1]):
                 keyList.append(storeName[ii]['4ID'])
-        smSet_all, smHdr_all = pbo_util.nostab_sys(storeName,storeData,[self.start_time,self.end_time],indx=keyList,mdyratio=mdyratio)
+        smSet_all, smHdr_all = pbo_util.nostab_sys(storeName,storeData,[self._start_time,self._end_time],indx=keyList,mdyratio=mdyratio)
 
         self.smSet_all = smSet_all
         self.smHdr_all = smHdr_all
@@ -215,7 +218,14 @@ def output(self):
                 data[ii] = self.smSet_all[ii]
                 info[ii] = self.smHdr_all[ii]
 
-        return DataWrapper(pd.Panel.from_dict(data, orient='minor'), geo_point, info)
+        if self.wrapper_type == 'series':
+            return DataWrapper(pd.Panel.from_dict(data, orient='minor'), geo_point, info)
+        elif self.wrapper_type == 'table':
+            return(DataPanelWrapper(pd.Panel.from_dict(data,orient='minor'), meta_data=info))
+        else:
+            print('... Invald wrapper type, defaulting to series ...')
+            return DataWrapper(pd.Panel.from_dict(data, orient='minor'), geo_point, info)
+
 
     def __str__(self):
         '''