From a10cb4851d79b7a7deff721b734a425759b6ce90 Mon Sep 17 00:00:00 2001 From: Tom Donoghue Date: Tue, 23 Feb 2021 13:35:39 -0500 Subject: [PATCH 1/7] add tresults & update check_dep --- fooof/core/modutils.py | 2 +- fooof/tests/conftest.py | 11 ++++++++++- fooof/tests/tutils.py | 11 +++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/fooof/core/modutils.py b/fooof/core/modutils.py index c342a52c9..a96f05012 100644 --- a/fooof/core/modutils.py +++ b/fooof/core/modutils.py @@ -177,6 +177,6 @@ def wrapped_func(*args, **kwargs): if not dep: raise ImportError("Optional FOOOF dependency " + name + \ " is required for this functionality.") - func(*args, **kwargs) + return func(*args, **kwargs) return wrapped_func return wrap diff --git a/fooof/tests/conftest.py b/fooof/tests/conftest.py index 65a8b00a9..f7d7094c3 100644 --- a/fooof/tests/conftest.py +++ b/fooof/tests/conftest.py @@ -8,7 +8,7 @@ from fooof.core.modutils import safe_import -from fooof.tests.tutils import get_tfm, get_tfg, get_tbands +from fooof.tests.tutils import get_tfm, get_tfg, get_tbands, get_tresults from fooof.tests.settings import BASE_TEST_FILE_PATH, TEST_DATA_PATH, TEST_REPORTS_PATH plt = safe_import('.pyplot', 'matplotlib') @@ -46,7 +46,16 @@ def tfg(): def tbands(): yield get_tbands() [email protected](scope='session') +def tresults(): + yield get_tresults() + @pytest.fixture(scope='session') def skip_if_no_mpl(): if not safe_import('matplotlib'): pytest.skip('Matplotlib not available: skipping test.') + [email protected](scope='session') +def skip_if_no_pandas(): + if not safe_import('pandas'): + pytest.skip('Pandas not available: skipping test.') diff --git a/fooof/tests/tutils.py b/fooof/tests/tutils.py index 51037468d..3572a3535 100644 --- a/fooof/tests/tutils.py +++ b/fooof/tests/tutils.py @@ -2,7 +2,10 @@ from functools import wraps +import numpy as np + from fooof.bands import Bands +from fooof.data import FOOOFResults from fooof.objs import FOOOF, FOOOFGroup from fooof.core.modutils import safe_import from fooof.sim.params import param_sampler @@ -43,6 +46,14 @@ def get_tbands(): return Bands({'theta' : (4, 8), 'alpha' : (8, 12), 'beta' : (13, 30)}) +def get_tresults(): + """Get a FOOOFResults objet, for testing.""" + + return FOOOFResults(aperiodic_params=np.array([1.0, 1.00]), + peak_params=np.array([[10.0, 1.25, 2.0], [20.0, 1.0, 3.0]]), + r_squared=0.97, error=0.01, + gaussian_params=np.array([[10.0, 1.25, 1.0], [20.0, 1.0, 1.5]])) + def default_group_params(): """Create default parameters for generating a test group of power spectra.""" From 673adeee59bcb43163cd0801b32cf4211acc9d18 Mon Sep 17 00:00:00 2001 From: Tom Donoghue Date: Tue, 23 Feb 2021 13:35:52 -0500 Subject: [PATCH 2/7] add converter funcs for data stores --- fooof/data/conversions.py | 106 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 fooof/data/conversions.py diff --git a/fooof/data/conversions.py b/fooof/data/conversions.py new file mode 100644 index 000000000..f2dbfeb7d --- /dev/null +++ b/fooof/data/conversions.py @@ -0,0 +1,106 @@ +"""Conversion functions for organizing model results into alternate representations.""" + +import numpy as np + +from fooof import Bands +from fooof.core.funcs import infer_ap_func +from fooof.core.info import get_ap_indices, get_peak_indices +from fooof.core.modutils import safe_import, check_dependency +from fooof.analysis.periodic import get_band_peak + +pd = safe_import('pandas') + +################################################################################################### +################################################################################################### + +def model_to_dict(fit_results, peak_org): + """Convert model fit results to a dictionary. + + Parameters + ---------- + fit_results : FOOOFResults + Results of a model fit. + peak_org : int or Bands + How to organize peaks. + If int, extracts the first n peaks. + If Bands, extracts peaks based on band definitions. + + Returns + ------- + dict + Model results organized into a dictionary. + """ + + fr_dict = {} + + # aperiodic parameters + for label, param in zip(get_ap_indices(infer_ap_func(fit_results.aperiodic_params)), + fit_results.aperiodic_params): + fr_dict[label] = param + + # periodic parameters + peaks = fit_results.peak_params + + if isinstance(peak_org, int): + + if len(peaks) < peak_org: + nans = [np.array([np.nan] * 3) for ind in range(peak_org-len(peaks))] + peaks = np.vstack((peaks, nans)) + + for ind, peak in enumerate(peaks[:peak_org, :]): + for pe_label, pe_param in zip(get_peak_indices(), peak): + fr_dict[pe_label.lower() + '_' + str(ind)] = pe_param + + elif isinstance(peak_org, Bands): + for band, f_range in peak_org: + for label, param in zip(get_peak_indices(), get_band_peak(peaks, f_range)): + fr_dict[band + '_' + label.lower()] = param + + # goodness-of-fit metrics + fr_dict['error'] = fit_results.error + fr_dict['r_squared'] = fit_results.r_squared + + return fr_dict + +@check_dependency(pd, 'pandas') +def model_to_dataframe(fit_results, peak_org): + """Convert model fit results to a dataframe. + + Parameters + ---------- + fit_results : FOOOFResults + Results of a model fit. + peak_org : int or Bands + How to organize peaks. + If int, extracts the first n peaks. + If Bands, extracts peaks based on band definitions. + + Returns + ------- + pd.Series + Model results organized into a dataframe. + """ + + return pd.Series(model_to_dict(fit_results, peak_org)) + + +@check_dependency(pd, 'pandas') +def group_to_dataframe(fit_results, peak_org): + """Convert a group of model fit results into a dataframe. + + Parameters + ---------- + fit_results : list of FOOOFResults + List of FOOOFResults objects. + peak_org : int or Bands + How to organize peaks. + If int, extracts the first n peaks. + If Bands, extracts peaks based on band definitions. + + Returns + ------- + pd.DataFrame + Model results organized into a dataframe. + """ + + return pd.DataFrame([model_to_dataframe(f_res, peak_org) for f_res in fit_results]) From 6a7ca58bcc97a3f95c28d46c9d3ca1834f406115 Mon Sep 17 00:00:00 2001 From: Tom Donoghue Date: Tue, 23 Feb 2021 13:36:10 -0500 Subject: [PATCH 3/7] add tests for data conversions --- fooof/tests/data/test_conversions.py | 53 ++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 fooof/tests/data/test_conversions.py diff --git a/fooof/tests/data/test_conversions.py b/fooof/tests/data/test_conversions.py new file mode 100644 index 000000000..4c50e653e --- /dev/null +++ b/fooof/tests/data/test_conversions.py @@ -0,0 +1,53 @@ +"""Tests for the fooof.data.conversions.""" + +from copy import deepcopy + +import numpy as np + +from fooof.core.modutils import safe_import +pd = safe_import('pandas') + +from fooof.data.conversions import * + +################################################################################################### +################################################################################################### + +def test_model_to_dict(tresults, tbands): + + out = model_to_dict(tresults, peak_org=1) + assert isinstance(out, dict) + assert 'cf_0' in out + assert out['cf_0'] == tresults.peak_params[0, 0] + assert not 'cf_1' in out + + out = model_to_dict(tresults, peak_org=2) + assert 'cf_0' in out + assert 'cf_1' in out + assert out['cf_1'] == tresults.peak_params[1, 0] + + out = model_to_dict(tresults, peak_org=3) + assert 'cf_2' in out + assert np.isnan(out['cf_2']) + + out = model_to_dataframe(tresults, peak_org=tbands) + assert 'alpha_cf' in out + +def test_model_to_dataframe(tresults, tbands, skip_if_no_pandas): + + for peak_org in [1, 2, 3]: + out = model_to_dataframe(tresults, peak_org=peak_org) + assert isinstance(out, pd.Series) + + out = model_to_dataframe(tresults, peak_org=tbands) + assert isinstance(out, pd.Series) + +def test_group_to_dataframe(tresults, tbands, skip_if_no_pandas): + + fit_results = [deepcopy(tresults), deepcopy(tresults), deepcopy(tresults)] + + for peak_org in [1, 2, 3]: + out = group_to_dataframe(fit_results, peak_org=peak_org) + assert isinstance(out, pd.DataFrame) + + out = group_to_dataframe(fit_results, peak_org=tbands) + assert isinstance(out, pd.DataFrame) From 7d00c362baece1fd4c5fc68868d7ba78cd754fcc Mon Sep 17 00:00:00 2001 From: Tom Donoghue Date: Tue, 23 Feb 2021 13:45:41 -0500 Subject: [PATCH 4/7] alias conversion functions to model objects --- fooof/objs/fit.py | 20 ++++++++++++++++++++ fooof/objs/group.py | 20 ++++++++++++++++++++ fooof/tests/objs/test_fit.py | 12 +++++++++++- fooof/tests/objs/test_group.py | 13 ++++++++++++- 4 files changed, 63 insertions(+), 2 deletions(-) diff --git a/fooof/objs/fit.py b/fooof/objs/fit.py index fba745d27..0055b6c77 100644 --- a/fooof/objs/fit.py +++ b/fooof/objs/fit.py @@ -78,6 +78,7 @@ from fooof.utils.data import trim_spectrum from fooof.utils.params import compute_gauss_std from fooof.data import FOOOFResults, FOOOFSettings, FOOOFMetaData +from fooof.data.conversions import model_to_dataframe from fooof.sim.gen import gen_freqs, gen_aperiodic, gen_periodic, gen_model ################################################################################################### @@ -716,6 +717,25 @@ def set_check_data_mode(self, check_data): self._check_data = check_data + def to_df(self, peak_org): + """Convert and extract the model results as a pandas object. + + Parameters + ---------- + peak_org : int or Bands + How to organize peaks. + If int, extracts the first n peaks. + If Bands, extracts peaks based on band definitions. + + Returns + ------- + pd.Series + Model results organized into a pandas object. + """ + + return model_to_dataframe(self.get_results(), peak_org) + + def _check_width_limits(self): """Check and warn about peak width limits / frequency resolution interaction.""" diff --git a/fooof/objs/group.py b/fooof/objs/group.py index 91abcee9e..033064611 100644 --- a/fooof/objs/group.py +++ b/fooof/objs/group.py @@ -21,6 +21,7 @@ from fooof.core.strings import gen_results_fg_str from fooof.core.io import save_fg, load_jsonlines from fooof.core.modutils import copy_doc_func_to_method, safe_import +from fooof.data.conversions import group_to_dataframe ################################################################################################### ################################################################################################### @@ -543,6 +544,25 @@ def print_results(self, concise=False): print(gen_results_fg_str(self, concise)) + def to_df(self, peak_org): + """Convert and extract the model results as a pandas object. + + Parameters + ---------- + peak_org : int or Bands + How to organize peaks. + If int, extracts the first n peaks. + If Bands, extracts peaks based on band definitions. + + Returns + ------- + pd.DataFrame + Model results organized into a pandas object. + """ + + return group_to_dataframe(self.get_results(), peak_org) + + def _fit(self, *args, **kwargs): """Create an alias to FOOOF.fit for FOOOFGroup object, for internal use.""" diff --git a/fooof/tests/objs/test_fit.py b/fooof/tests/objs/test_fit.py index 87907d11a..7fd0b12bd 100644 --- a/fooof/tests/objs/test_fit.py +++ b/fooof/tests/objs/test_fit.py @@ -12,9 +12,12 @@ from fooof.core.items import OBJ_DESC from fooof.core.errors import FitError from fooof.core.utils import group_three +from fooof.core.modutils import safe_import +from fooof.core.errors import DataError, NoDataError, InconsistentDataError from fooof.sim import gen_freqs, gen_power_spectrum from fooof.data import FOOOFSettings, FOOOFMetaData, FOOOFResults -from fooof.core.errors import DataError, NoDataError, InconsistentDataError + +pd = safe_import('pandas') from fooof.tests.settings import TEST_DATA_PATH from fooof.tests.tutils import get_tfm, plot_test @@ -425,3 +428,10 @@ def test_fooof_check_data(): # Model fitting should execute, but return a null model fit, given the NaNs, without failing tfm.fit() assert not tfm.has_model + +def test_fooof_to_df(tfm, tbands, skip_if_no_pandas): + + df1 = tfm.to_df(2) + assert isinstance(df1, pd.Series) + df2 = tfm.to_df(tbands) + assert isinstance(df2, pd.Series) diff --git a/fooof/tests/objs/test_group.py b/fooof/tests/objs/test_group.py index 7a90cfd56..39213be64 100644 --- a/fooof/tests/objs/test_group.py +++ b/fooof/tests/objs/test_group.py @@ -9,10 +9,14 @@ import numpy as np from numpy.testing import assert_equal -from fooof.data import FOOOFResults from fooof.core.items import OBJ_DESC +from fooof.core.modutils import safe_import +from fooof.core.errors import DataError, NoDataError, InconsistentDataError +from fooof.data import FOOOFResults from fooof.sim import gen_group_power_spectra +pd = safe_import('pandas') + from fooof.tests.settings import TEST_DATA_PATH from fooof.tests.tutils import default_group_params, plot_test @@ -349,3 +353,10 @@ def test_fg_get_group(tfg): # Check that the correct results are extracted assert [tfg.group_results[ind] for ind in inds1] == nfg1.group_results assert [tfg.group_results[ind] for ind in inds2] == nfg2.group_results + +def test_fg_to_df(tfg, tbands, skip_if_no_pandas): + + df1 = tfg.to_df(2) + assert isinstance(df1, pd.DataFrame) + df2 = tfg.to_df(tbands) + assert isinstance(df2, pd.DataFrame) From b8afe5410277557e882e781ef09603e8b7e3c5c6 Mon Sep 17 00:00:00 2001 From: Tom Donoghue Date: Tue, 23 Feb 2021 13:51:07 -0500 Subject: [PATCH 5/7] fix typo that was running wrong func in tests --- fooof/tests/data/test_conversions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fooof/tests/data/test_conversions.py b/fooof/tests/data/test_conversions.py index 4c50e653e..08ab30bde 100644 --- a/fooof/tests/data/test_conversions.py +++ b/fooof/tests/data/test_conversions.py @@ -29,7 +29,7 @@ def test_model_to_dict(tresults, tbands): assert 'cf_2' in out assert np.isnan(out['cf_2']) - out = model_to_dataframe(tresults, peak_org=tbands) + out = model_to_dict(tresults, peak_org=tbands) assert 'alpha_cf' in out def test_model_to_dataframe(tresults, tbands, skip_if_no_pandas): From ec8c0e7ade9d6a9fd7125db79ed59fac78aac478 Mon Sep 17 00:00:00 2001 From: Tom Donoghue Date: Tue, 23 Feb 2021 13:55:05 -0500 Subject: [PATCH 6/7] add pandas as a listed optional requirement --- optional-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/optional-requirements.txt b/optional-requirements.txt index 7ea3f6a80..4df6a015c 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -1,2 +1,3 @@ matplotlib -tqdm \ No newline at end of file +tqdm +pandas \ No newline at end of file From 7ba98343e90805710f9661d841911d974075b56e Mon Sep 17 00:00:00 2001 From: Tom Donoghue Date: Thu, 29 Jun 2023 15:20:11 -0700 Subject: [PATCH 7/7] add pandas to readme list of optional dependencies --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 5c36cb0a0..44a71eae3 100644 --- a/README.rst +++ b/README.rst @@ -82,6 +82,7 @@ There are also optional dependencies, which are not required for model fitting i - `matplotlib `_ is needed to visualize data and model fits - `tqdm `_ is needed to print progress bars when fitting many models +- `pandas `_ is needed to for exporting model fit results to dataframes - `pytest `_ is needed to run the test suite locally We recommend using the `Anaconda `_ distribution to manage these requirements.