Skip to content

Add filter feature selection using Pearson correlation #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mandjevant wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add filter feature selection using Pearson correlation #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions pyfume/BuildTakagiSugeno.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,9 @@ def __init__(self, datapath=None, dataframe=None, nr_clus=2, variable_names=None
if kwargs['feature_selection'] == 'wrapper':
self.selected_feature_indices, self.variable_names=fs.wrapper()
elif kwargs['feature_selection'] == 'fst-pso':
self.selected_feature_indices, self.variable_names, self.nr_clus= fs.fst_pso_feature_selection(max_iter=kwargs['fs_max_iter'])
self.selected_feature_indices, self.variable_names, self.nr_clus= fs.fst_pso_feature_selection(max_iter=kwargs['fs_max_iter'])
elif kwargs['feature_selection'] == 'filter':
self.selected_feature_indices, self.variable_names = fs.filter()
self.x_train = self.x_train[:, self.selected_feature_indices]
self.x_test = self.x_test[:, self.selected_feature_indices]

Expand Down Expand Up @@ -252,7 +254,9 @@ def __init__(self, datapath=None, dataframe=None, nr_clus=2, variable_names=None
if kwargs['feature_selection'] == 'wrapper':
self.selected_feature_indices, self.selected_variable_names=fs.wrapper(feature_selection_stop=0.05)
elif kwargs['feature_selection'] == 'fst-pso' or kwargs['feature_selection'] == 'fstpso':
self.selected_feature_indices, self.selected_variable_names, self.nr_clus= fs.fst_pso_feature_selection(max_iter=kwargs['fs_max_iter'])
self.selected_feature_indices, self.selected_variable_names, self.nr_clus= fs.fst_pso_feature_selection(max_iter=kwargs['fs_max_iter'])
elif kwargs['feature_selection'] == 'filter':
self.selected_feature_indices, self.selected_variable_names = fs.filter()

self.x_train = self.x_train[:, self.selected_feature_indices]
self.x_test = self.x_test[:, self.selected_feature_indices]
Expand Down Expand Up @@ -348,7 +352,9 @@ def __init__(self, datapath=None, dataframe=None, nr_clus=2, variable_names=None
if kwargs['feature_selection'] == 'wrapper':
self.selected_feature_indices, self.variable_names=fs.wrapper()
elif kwargs['feature_selection'] == 'fst-pso':
self.selected_feature_indices, self.variable_names, self.nr_clus= fs.fst_pso_feature_selection(max_iter=kwargs['fs_max_iter'])
self.selected_feature_indices, self.variable_names, self.nr_clus= fs.fst_pso_feature_selection(max_iter=kwargs['fs_max_iter'])
elif kwargs['feature_selection'] == 'filter':
self.selected_feature_indices, self.variable_names = fs.filter()
self.x_train = self.x_train[:, self.selected_feature_indices]

elif kwargs['feature_selection'] == None:
Expand Down
54 changes: 51 additions & 3 deletions pyfume/FeatureSelection.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,57 @@ def fst_pso_feature_selection(self,max_iter=100, min_clusters=2, max_clusters=10

return selected_features, varnams, optimal_number_clusters

# def fun(self, particle):
# return sum(particle)

@staticmethod
def transpose(arr: list) -> list:
return [[arr[j][i] for j, _ in enumerate(arr)] for i, _ in enumerate(arr[0])]

def filter(self, min_corr=0.2, max_corr=1.0, max_pvalue=0.05):
"""
Performs feature selection using the Pearson correlation filter method.
Filters on both p value and Pearson correlation.

Args:
min_corr: Minimum correlation value for feature to be selected.
max_corr: Maximum correlation value for feature to be selected.
max_pvalue: Maximum p-value to determine statistical significance.

Returns:
Tuple containing (selected_feature_indices, selected_feature_names)
- selected_feature_indices: The indices of the selected features.
- selected_feature_names: The names of the selected features.

"""
from scipy.stats import pearsonr

transposed_matrix = FeatureSelector.transpose(self.dataX)

pearson_corr_list = [[pearsonr(transposed_matrix[inx], self.dataY), (inx, self.variable_names[inx])]
for inx, _ in enumerate(transposed_matrix)]

for corr in pearson_corr_list.copy():
if abs(corr[0][1]) > max_pvalue or abs(corr[0][0]) < min_corr or abs(corr[0][0]) > max_corr:
pearson_corr_list.remove(corr)

selected_features = [selected_feature[1][0] for selected_feature in pearson_corr_list]
selected_feature_names = [selected_feature[1][1] for selected_feature in pearson_corr_list]

# Calculate the performance of the selected features. Alike the wrapper method.
ds = DataSplitter()
x_feat, y_feat, x_val, y_val = ds.holdout(self.dataX, self.dataY)

feat = x_feat[:, selected_features]
x_validation = x_val[:, selected_features]

performance = self._evaluate_feature_set(x_data=feat, y_data=y_feat, x_val=x_validation, y_val=y_val,
nr_clus=self.nr_clus, var_names=selected_feature_names,
model_order=self.model_order,
performance_metric=self.performance_metric)

print('The selected features have a', self.performance_metric, 'of:', performance)
print('The following features were selected:', selected_feature_names)

return selected_features, selected_feature_names

def _function(self, particle, arguments, verbose=True, **kwargs):
from itertools import compress
if self.nr_clus == None:
Expand Down