diff --git a/.claude/rules/status_conduct.md b/.claude/rules/status_conduct.md new file mode 100644 index 0000000..b7b4ffe --- /dev/null +++ b/.claude/rules/status_conduct.md @@ -0,0 +1,23 @@ +--- +paths: + - ".status/**" +--- + +# Working in .status/ + +- `.status/` is gitignored: the only copy is on this machine. Never delete, + move, or rewrite a file here without asking first. Appending is fine. +- Every markdown file written here opens with a `For humans:` summary - three + or four sentences at the very top: what the file is and what a person needs + to take from it. +- Nothing new is created at the `.status/` root. New material goes in + `plans/`, `prompts/`, `notes/`, or `scratch/`. +- Only `plans/` and `prompts/` are edited in place. `notes/` is write-once. + `decisions.md` is append-only - never rewrite an entry. +- Temporary scripts, experiments, and one-off test files go in + `.status/scratch/` - never the repository root. `scratch/` may hold any + file type and is deleted unread; everywhere else is markdown only. +- Do not read `archive/` unless a file is named for you. +- Filenames: lowercase ASCII, `_` as separator. Notes are + `YYYY-MM-DD_slug.md` (date first); plans and prompts are `slug.md` - no + date, no status word in the name. diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..c0da300 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-settings.json", + "permissions": { + "blockReadsOutsideWorkingDirectories": true, + "allow": [ + "Bash(python --version)", + "Bash(unzip -l *)", + "Bash(python -m sphinx *)", + "Bash(python docs/patch_matlabdomain.py)", + "Bash(git status)", + "Bash(git status *)", + "Bash(git log)", + "Bash(git log *)", + "Bash(git diff)", + "Bash(git diff *)", + "Bash(git branch)", + "Bash(git ls-files *)", + "Bash(ls)", + "Bash(ls *)", + "Bash(wc *)" + ], + "ask": [ + "Bash(matlab -batch *)", + "Bash(git add *)", + "Bash(git commit *)", + "Bash(git checkout -- *)", + "Bash(git restore *)", + "Bash(git stash *)" + ], + "deny": [ + "Read(.env)", + "Read(.envrc)", + "Read(.status/archive/**)", + "Bash(rm -rf *)", + "Bash(rm -r *)", + "Bash(git push)", + "Bash(git push *)", + "Bash(git reset --hard *)", + "Bash(pip install *)", + "Bash(python -m pip install *)", + "Bash(curl *)", + "Bash(wget *)" + ] + } +} diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..85d8836 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,46 @@ +# Set default behavior to automatically normalize line endings to LF +* text=auto eol=lf + +# Explicitly declare text files you want to always be normalized and converted to LF on checkout +*.xml text eol=lf +*.mediawiki text eol=lf +*.tsv text eol=lf +*.md text eol=lf +*.rst text eol=lf +*.txt text eol=lf +*.m text eol=lf +*.tex text eol=lf +*.py text eol=lf +*.sh text eol=lf +*.yml text eol=lf +*.yaml text eol=lf +*.json text eol=lf +*.toml text eol=lf +*.svg text eol=lf +*.css text eol=lf +*.js text eol=lf +*.html text eol=lf + +# Denote all files that are truly binary and should not be modified +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary +*.pdf binary +*.zip binary +*.set binary +*.fdt binary +*.gz binary +*.tar binary +*.mp3 binary +*.mp4 binary +*.mov binary +*.avi binary +*.exe binary +*.dll binary +*.so binary +*.dylib binary +*.class binary +*.jar binary +*.war binary diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..b8fa164 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,18 @@ +# EEG-Clean-Tools + +**`AGENTS.md` at the repository root is the instruction set for this project. +Read it before answering, and follow it.** + +This file is a pointer and duplicates nothing. One source, several pointers: a +rule stated in two files is a rule that will disagree with itself. + +Path-specific instructions live in `.github/instructions/*.instructions.md` +and load automatically when the files being worked on match their `applyTo` +glob - for example, `status_conduct.instructions.md` applies under `.status/`. +Nothing needs to reference them; this note exists so a reader knows they are +there. + +Machine-specific facts - interpreter, local paths, cache locations - are in +`.status/local-environment.md`, which is gitignored: read it when it is there +and ignore its absence when it is not. No committed file in this repository +may contain a local path or a drive letter. diff --git a/.github/instructions/status_conduct.instructions.md b/.github/instructions/status_conduct.instructions.md new file mode 100644 index 0000000..389e841 --- /dev/null +++ b/.github/instructions/status_conduct.instructions.md @@ -0,0 +1,22 @@ +--- +applyTo: ".status/**" +--- + +# Working in .status/ + +- `.status/` is gitignored: the only copy is on this machine. Never delete, + move, or rewrite a file here without asking first. Appending is fine. +- Every markdown file written here opens with a `For humans:` summary - three + or four sentences at the very top: what the file is and what a person needs + to take from it. +- Nothing new is created at the `.status/` root. New material goes in + `plans/`, `prompts/`, `notes/`, or `scratch/`. +- Only `plans/` and `prompts/` are edited in place. `notes/` is write-once. + `decisions.md` is append-only - never rewrite an entry. +- Temporary scripts, experiments, and one-off test files go in + `.status/scratch/` - never the repository root. `scratch/` may hold any + file type and is deleted unread; everywhere else is markdown only. +- Do not read `archive/` unless a file is named for you. +- Filenames: lowercase ASCII, `_` as separator. Notes are + `YYYY-MM-DD_slug.md` (date first); plans and prompts are `slug.md` - no + date, no status word in the name. diff --git a/.github/workflows/deploy-docs.yaml b/.github/workflows/deploy-docs.yaml new file mode 100644 index 0000000..656399b --- /dev/null +++ b/.github/workflows/deploy-docs.yaml @@ -0,0 +1,88 @@ +name: Deploy Documentation + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +# Read-only by default. Only the deploy job, which runs only on pushes to +# master, gets the Pages and OIDC write permissions; the build job runs code +# from the branch (docs/conf.py, the patch script) and stays read-only. +permissions: + contents: read + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + pages: read # actions/configure-pages reads the Pages site settings + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + with: + python-version: '3.10' + enable-cache: true + cache-dependency-glob: "**/pyproject.toml" + + - name: Create virtual environment + run: | + uv venv --clear .venv + echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH + + - name: Install dependencies + run: uv pip install -e ".[docs]" + + - name: Patch sphinxcontrib-matlabdomain + run: python docs/patch_matlabdomain.py + + - name: Configure Git for GitHub Pages + run: | + git config user.name github-actions + git config user.email github-actions@github.com + + - name: Build documentation + run: | + sphinx-build -b html docs docs/_build/html + + - name: Setup Pages + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6 + + - name: Upload artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 + with: + path: ./docs/_build/html + + #------------------------------------------------ + # Deploy Job: Deploys the built site + #------------------------------------------------ + deploy: + # This job depends on the 'build' job completing successfully + needs: build + # Only deploy when pushing to master, not on pull requests + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + permissions: + pages: write + id-token: write + runs-on: ubuntu-latest + + # Specify the deployment environment + environment: + name: github-pages + # The URL will be automatically set by the deployment step's output + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Deploy to GitHub Pages + # This is the official action for deploying the artifact to GitHub Pages + id: deployment + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5 diff --git a/.gitignore b/.gitignore index 21229f7..24c1e80 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,25 @@ -*.asv \ No newline at end of file +*.asv + +# Personal per-repo Claude Code notes (imports .status/local-environment.md) +CLAUDE.local.md + +# Personal per-repo settings (absolute paths, claudeMdExcludes) +.claude/settings.local.json + +# Real tokens and machine values; .env.example is the committed documentation +.env + +# Working notes, plans, prompts, and decision records. Deliberately NOT +# committed: these repos are public and .status/ holds half-formed thinking. +# The consequence is that it lives on one machine only and is absent from +# fresh clones and from `claude --worktree` worktrees. +.status/ + +# Documentation build and its Python toolchain +docs/_build/ +.venv/ +__pycache__/ +*.egg-info/ + +# Output of the example scripts +PrepPipeline/examples/output/ diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..8958373 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,8 @@ +{ + "files.eol": "\n", + "files.insertFinalNewline": true, + "files.trimTrailingWhitespace": true, + + "python.testing.pytestEnabled": false, + "python.testing.unittestEnabled": false +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..f87f4c3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,69 @@ +# EEG-Clean-Tools + +Purpose: the PREP pipeline - standardized early-stage EEG preprocessing (boundary handling, detrending, line-noise removal, robust average referencing, bad-channel detection and interpolation, reporting) as a MATLAB toolbox and an EEGLAB plugin. + +Not in scope: EEGLAB itself, which PREP runs inside and depends on, and the downstream analysis PREP deliberately leaves open (final high-pass filtering, ICA). + +## Commands + +Test framework: none. There is no test suite. Do not add a suite as a side effect of other work. + +- Smoke check (no EEGLAB needed): `matlab -batch "addpath(genpath('PrepPipeline')); disp(getPrepVersion())"` - prints the version string, for example `PrepPipeline0.57.0` +- Run standalone: add `PrepPipeline` and its subfolders to the MATLAB path, then `[EEG, params, computationTimes] = prepPipeline(EEG, params)` on an EEGLAB `EEG` structure with channel locations. Needs EEGLAB and the Signal Processing Toolbox on the path. +- Run as a plugin: unzip `EEGLABPlugin/PrepPipeline.zip` into EEGLAB's `plugins/` folder; the menu entry is Tools -> Run PREP pipeline. +- Check the plugin zip: `unzip -l EEGLABPlugin/PrepPipeline.zip` +- Install the docs toolchain: `python -m venv --clear .venv`, activate it, then `python -m pip install -e ".[docs]"` and `python docs/patch_matlabdomain.py` (required after every install of `sphinxcontrib-matlabdomain`; it fixes a Sphinx 7+ incompatibility in that package). Locally, use pip and the activated `.venv`, never `uv` or `uvx`: uv misbehaves on Windows. The GitHub Actions workflows use uv, and that stays. +- Build docs: `python -m sphinx -b html docs docs/_build/html` - `.github/workflows/deploy-docs.yaml` runs the same build and publishes it to GitHub Pages on pushes to `master` + +## Layout + +- `PrepPipeline/` - entry points: `prepPipeline.m`, `pop_prepPipeline.m` (EEGLAB GUI wrapper), `prepPostProcess.m`, `prepReport.m`, `publishPrepReport.m`, `eegplugin_prepPipeline.m` (EEGLAB menu registration) +- `PrepPipeline/utilities/` - the algorithms (`removeTrend`, `cleanLineNoise`, `performReference`, `findNoisyChannels`, defaults, version); `chronux_2_modified/` is vendored third-party code (GPL v2; see the licensing table in `README.md`) +- `PrepPipeline/reporting/` - report and collection-statistics functions +- `PrepPipeline/interface/` - the EEGLAB parameter GUIs +- `PrepPipeline/derived/`, `PrepPipeline/examples/`, `PrepPipeline/extracted/` - scripts built on the pipeline +- `EEGLABPlugin/` - the released plugin as a zip +- `docs/` - Sphinx source for the documentation site (MyST markdown and `.rst`); images in `docs/_static/images/`. `pyproject.toml` exists only to declare the docs toolchain. +- `CHANGELOG.md` - release history +- `.status/` - working notes. Gitignored; local to each machine. + +## Conventions that differ from defaults + +- **ASCII only** in prose, code, comments, and filenames: `-` not em or en dashes, `->` not arrows, `...` not an ellipsis character, straight quotes. Exception: genuine data (author names, dataset titles, recorded API responses) keeps whatever characters it actually contains. +- Markdown headers are sentence case: capitalize only the first word, proper nouns, and acronyms (PREP, EEG, EEGLAB, MATLAB). +- MATLAB: camelCase for functions. The file name must match the function name, so `.m` files keep their mixed case. +- The default branch is `master`, not `main`. + +## Rules that are easy to get wrong + +- The version exists in three places that must agree: `PrepPipeline/utilities/getPrepVersion.m` (the change log that `getPrepVersion` returns), the zip name under `EEGLABPlugin/`, and `CHANGELOG.md`. Change all three together. +- Do not reformat, lint, or ASCII-clean vendored code under `PrepPipeline/utilities/chronux_2_modified/`. +- `docs/api.rst` pulls each function's help text from the comment block right after its `function` line. `docs/conf.py` shows that text preformatted, exactly as MATLAB `help` prints it, so write help for `help`, not as reStructuredText. Help placed above the `function` line does not appear there, though MATLAB `help` still finds it. Functions at the root of `PrepPipeline/` need `.. mat:currentmodule:: .` before their `mat:autofunction` directives. +- Do not change the signature of an entry-point function without discussion; EEGLAB and user scripts call them directly, and `pop_prepPipeline` writes the call into EEGLAB history. + +## Related repositories + +Referred to by name; none is vendored here. + +- `eeglab` - the MATLAB toolbox PREP runs in, required at runtime. +- `hed-matlab` - the model for this repository's layout and documentation setup. + +## Where the thinking lives + +`.status/` is gitignored, so it exists only on the machine that wrote it and never in a fresh clone or worktree. + +- `.status/README.md` - the index. Read this first; it lists what is active. +- `.status/decisions.md` - why things are the way they are. Read before proposing structural changes. Append entries; never rewrite one. +- `.status/plans/*.md` - active plans. Check the `Status:` header and the `[ ]` / `[x]` markers before starting work. +- `.status/local-environment.md` - this machine's paths, interpreter, and quirks. Tool-agnostic. Never copy its contents into a committed file. +- IMPORTANT: do not read `.status/archive/` unless a file is named for you. Nothing new is created at the `.status/` root. + +## Working agreements + +- IMPORTANT: every file written to `.status/` opens with a `For humans:` summary - three or four sentences, at the very top: what the file is and what a person needs to take from it. The same applies to a long answer in a session: lead with the conclusion. +- IMPORTANT: temporary scripts, experiments, and one-off test files go in `.status/scratch/` - **never the repository root**. Delete them when the experiment ends; anything in `scratch/` may be deleted unread. +- IMPORTANT: never delete or rewrite a file under `.status/` without asking first. Appending is fine. +- For a change spanning more than three files, write a plan to `.status/plans/` and stop for review before editing. +- When you are guessing about an external API or data format, say so explicitly rather than assuming. +- Show evidence, not assertions: the command you ran and its actual output. +- Do not commit, push, or create branches unless asked. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6a86606 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,221 @@ +# Changelog + +Release history of the PREP pipeline, newest first. The current version is also reported by `getPrepVersion` (`PrepPipeline/utilities/getPrepVersion.m`). + +## Unreleased + +* Fixed an off-by-one in robust referencing: it now performs at most `maxReferenceIterations` passes; it used to perform one more, so results change for recordings that never converge + +## Version 0.57.0 - Released 3/31/2025 + +* Modified to work with modified EEGLAB GUI Builder +* Modified reporting to not clutter workspace + +## Version 0.56.0 - Released 8/01/2021 + +* Corrected parfor failure when channel number not consecutive +* Fixed missing badChannelsFromDropout in updateBadChannels issue#28 + +## Version 0.55.4 - Released 7/26/2020 + +* Correctly restored EEGLAB options after execution +* Added functions to output errors from etc.noiseDetection +* Corrected findpeaks naming conflict in Chronux +* Post process does not execute if Prep had errors + +## Version 0.55.3 - Released 10/19/2017 + +* Fixed issue with interpolated channels when interpolation order is pre-process +* Fixed issue with correct removal of interpolated channels during post-processing +* Reordered preprocessing and report buttons on master GUI + +## Version 0.55.2 - Released 08/18/2017 + +* Fixed undefined reference to referenceOut in prepPipeline post process + +## Version 0.55.1 - Released 06/03/2017 + +* Wrote printListCompressed to display channels more compactly +* Put in a MATLAB version check because legend titles not supported in 2014b +* Fixed spacing on output of interpolated channel numbers + +## Version 0.55.0 - Released 05/29/2017 + +* Changed the EEG.etc.noiseDetection structure to contain removed channels and interpolated channels for easier access +* Fixed reporting to work when bad channels have been removed +* Added original channel labels to EEG.etc.noiseDetection for ease in reporting +* Added Blasst as an unsupported line noise removal option +* Moved legend of spectrum to right, put in checks for removed channels +* Corrected bug in smoothing in cleanline +* Corrected several reporting issues +* Default behavior now outputs errors to command line in addition to logging +* Renamed several functions to make naming scheme consistent +* Started supporting changelog in versions +* Fixed bug in struct2str and improved com return on pop_prepPipeline + +## Version 0.52 - Released + +* Modified code to handle EEG structures with empty EEG.error. +* Performed additional minor cleanup. + +## Version 0.51 - Not released + +* Developing bad window visualization plugin for EEGLAB + +## Version 0.50 - Released + +* Made several cleanup modifications to ready for release. + +## Version 0.48 - (Not released -- version 0.47 with EEGLAB integration) + +* Integrated EEGLAB plugin +* Changed the default structure value field name from defaults.default to + default.value and propagated the change +* Changed default names of line noise and global trend to linenoise and + globaltrend +* Modified the resampling step to allow an option low pass filter to remove + downsampling artifacts just below Nyquist frequency. + +## Version 0.47 - (Not released -- version 0.46 with additional changes) + +* Minor refactoring of performReference to avoid 1 extra filtering operation --- + should not reflect results. +* Also added average and specific referencing methods -- not tested as yet. + +## Version 0.46 - (Not released - version 0.45 with additional changes) + +* Fixed remapping of bad evaluation channels into original channel numbers + (relevant when there are none EEG channels interspersed in the channel + locations. +* Passed detrend information in reference structure to allow detrending + with other than the defaults +* Corrected several channel mapping issues in the reporting. + +## Version 0.45 - (Not released - version 0.44 with additional changes) + +* Refactored report to allow statistics to be gathered from noisy structures + +## Version 0.44 - (Not released - version 0.43 with additional changes) + +* Corrected a minor issue with reporting -- difference between robust + and ordinary reference had axes reversed. +* Updated to run with plotting compatible with MATLAB 2014b +* Added box on to cummulative plots. + +## Version 0.43 - (Not released - version 0.42 with additional changes) + +* Corrected a minor issue with reporting -- mean scalp correlation map for + beforeInterpolation was plotting the Original data rather than the + beforeInterpolation data. + +## Version 0.42 - (Not released - version 0.41 with additional changes) + +* Added default line frequencies as multiples of 60 up to half nyquist. + +## Version 0.41 - (Not released - version 0.40 with additional changes) + +* Replaced default method with channel forgetting and median initialization +* Converted EEG to double at the beginning of the pipeline +* Added a noisyStatisticsForInterpolation field to the reference reporting + structure. + +## Version 0.40 - (Not yet released - major change in strategy) + +* Changed the name from StandardLevel2 to PrepPipeline +* Implemented the HP filter-free strategy +* Added a keepFiltered version -- if false (the default) the data in the + repository is not high pass filtered +* Added an option for removing global trend +* Incorporated the different reference schemes into a single performReference + +## Version 0.28 - (Not yet released) + +* Changed the name of the noisyParameter structure in EEG.etc to + noiseDetection. This is a major change with corresponding change + in ESS. +* Added a specificReferenceChannels field to reference structure +* Changed the averageReference field name to referenceSignal in reference + structure +* Included a referenceType field in the reference structure (this + can be 'robust', 'average', or 'specific') +* Eliminated the don't interpolateHFChannels flag. +* Added routines to do specificReference (mastoid or average) +* Modified showSpectrum to return the spectra of all of the channels. +* Detrending at 0.2 Hz has replaced FIR filtering as default trend removal. + +## Version 0.27 - Released 1/7/2015 + +* Correct version of bug fix in cleanLineNoise -- watch that single + precision conversion! + +## Version 0.26 - Released 1/7/2015 + +* Release to fix bug in cleanLineNoise --- channels that are not + lineNoiseChannels were set to zero rather than being carried forward. + +## Version 0.25 - Released 1/5/2015 (major) + +* Removed saving of temporary file after line noise removal +* Fixed report of relative reference +* Modified findNoisyChannels to exclude NaN and constant channels + from noisyChannel thresholding, but to designate them as bad channels +* Moved resampling step before high pass filter +* Assigned return values in a separate step +* Put error check in ShowSpectrum when invalid data is invalid +* Correct minor issues with PlotScalpMap +* Added extractReferenceStatistics -- which extracts summary statistics + for an entire archive. +* Added iterations on the remove robust reference +* Added a summary reporting scheme for spotting problematic datasets. + +## Version 0.24 - Released 12/7/2104 (major) + +* Fixed channel selection bug in showSpectrum +* Added error handling for failures in standardLevel2Pipeline +* Added error reporting for failures +* Corrected time scale on visualization of difference between + robust and mean reference +* Added channel labels as well as numbers to spectrum visualization +* Fixed major bug in robustReference so that original signal is rereferenced +* Revised and expanded the reporting + +## Version 0.23 - Released 11/13/2014 + +* Removed the channel locations and channel information from noisyOut + because it is already in the reference structure at top level. +* Added reporting of average fraction of channels bad in windows. +* Added first version of hdf5support -- rewrites the noisyParameters + to an HDF5 file. + +## Version 0.22 - Released 11/9/2014 + +* Revised the method of computing the windowed channel deviations +* Added summary reporting functions +* Added a check to only perform ransac when sufficiently good channels + are available +* Added check to only perform ransac when channel locations are available +* Fixed the input parameter structure on findNoisyChannels +* Added the infrastructure for the summary of all datasets + +## Version 0.21 - Released 10/30/2104 + +* Removed any reference to chanlocs in highPassFilter +* Full integration with ESS Study Level 2 code +* Preliminary version of Standard Level 2 Report finalized (gives pdf) + +## Version 0.20 - Released 10/18/2014 + +* Converted standardLevel2Pipeline to a function +* Moved the computationTimes structure to standardLevel2Pipeline so that +it is returned. + +## Version 0.19 - Released 10/16/2014 + +* Refactored name is also included in the params structure. +* Renamed rereferencedChannels as channelsToBeReferenced to agree with ESS. + +## Version 0.18 - Released 10/15/2014 + +* Refactored so that all input to the pipeline is in a single params structure. +* Fixed the HF noise reporting windows and several minor bugs +* Added visualizations to show number of bad channels in each window diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..39297d0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,7 @@ +@AGENTS.md + +# Claude Code + +- Path-scoped rules live in `.claude/rules/` and load automatically when a session touches files matching their `paths:` globs - for example, `status_conduct.md` applies under `.status/`. Nothing needs to reference them; this note exists so a reader knows they are there. +- `CLAUDE.local.md` (gitignored) holds notes about this machine's Claude Code setup and imports `.status/local-environment.md`, which is where machine facts live for every tool, not just this one. +- After changing either import, run `/context` and confirm the file appears under **Memory files**. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3912109 --- /dev/null +++ b/LICENSE @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/PrepPipeline/derived/highPassAndICALinux.m b/PrepPipeline/derived/highPassAndICALinux.m deleted file mode 100644 index fad8057..0000000 --- a/PrepPipeline/derived/highPassAndICALinux.m +++ /dev/null @@ -1,39 +0,0 @@ -function EEG = highPassAndICA(EEG, varargin) -% Perform a high-pass filter and ICA on data that has been Prepped -try - params = vargin2struct(varargin); - if isfield(EEG.etc, 'noiseDetection') && ... - isfield(EEG.etc.noiseDetection, 'detrend') && ... - ~isfield(params, 'detrendChannels') - params.detrendChannels = EEG.etc.noiseDetection.detrend.detrendChannels; - end - [EEG, detrend] = removeTrend(EEG, params); - EEG.etc.highPassAndICA.detrend = detrend; - - %% Perform ICA on the data - if isfield(EEG.etc, 'noiseDetection') && ... - isfield(EEG.etc.noiseDetection, 'reference') - referenceChannels = ... - EEG.etc.noiseDetection.reference.referenceChannels; - interpolatedChannels = ... - EEG.etc.noiseDetection.reference.interpolatedChannels.all; - channelsLeft = setdiff(referenceChannels, interpolatedChannels); - pcaDim = length(channelsLeft) - 1; - fprintf('%s: pcaDim = %d\n', EEG.setname, pcaDim); - EEG = pop_runica(EEG, 'icatype', 'cudaica', 'extended', 1, ... - 'chanind', referenceChannels, 'pca', pcaDim); - EEG.etc.highPassAndICA.ICA = ... - ['extended infomax with pca Dim ' num2str(pcaDim)]; - EEG.etc.noiseDetection.reference = ... - cleanupReference(EEG.etc.noiseDetection.reference); -% else -% EEG = pop_runica(EEG, 'icatype', 'runica', 'extended', 1); -% EEG.etc.highPassAndICA.ICA = 'runica extended infomax'; - end - -catch mex - errorMessages.highPassAndICA = ['failed highPassAndICA: ' getReport(mex)]; - errorMessages.status = 'unprocessed'; - EEG.etc.highPassAndICA.errors = errorMessages; - fprintf(2, '%s\n', errorMessages.highPassAndICA); -end \ No newline at end of file diff --git a/PrepPipeline/eegplugin_prepPipeline.m b/PrepPipeline/eegplugin_prepPipeline.m index 0bf3f1a..9919dc9 100644 --- a/PrepPipeline/eegplugin_prepPipeline.m +++ b/PrepPipeline/eegplugin_prepPipeline.m @@ -1,43 +1,43 @@ -% eegplugin_prepPipeline() - a wrapper to the prepPipeline, which does early stage -% -% Usage: -% >> eegplugin_prepPipeline(fig, try_strings, catch_strings); -% -% see also: prepPipeline - -% Author: Kay Robbins, with contributions from Nima Bigdely-Shamlo, Tim Mullen, Christian Kothe, and Cassidy Matousek. - -% This program is free software; you can redistribute it and/or modify -% it under the terms of the GNU General Public License as published by -% the Free Software Foundation; either version 2 of the License, or -% (at your option) any later version. -% -% This program is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -% GNU General Public License for more details. -% -% You should have received a copy of the GNU General Public License -% along with this program; if not, write to the Free Software -% Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -%function eegplugin_clean_rawdata(fig,try_strings,catch_strings) - - -% eegplugin_prepPipeline() - the PREP pipeline plugin -function vers = eegplugin_prepPipeline(fig, trystrs, catchstrs) - -%% Add path to prepPipeline subdirectories if not in the list -tmp = which('getPrepDefaults'); -if isempty(tmp) - myPath = fileparts(which('prepPipeline')); - addpath(genpath(myPath)); -end -vers = getPrepVersion(); - -% create menu -comprep = [trystrs.no_check '[EEG LASTCOM] = pop_prepPipeline(EEG);' catchstrs.new_and_hist]; -menu = findobj(fig, 'tag', 'tools'); -uimenu( menu, 'label', 'Run PREP pipeline', 'callback', comprep, ... - 'separator', 'on'); - +% eegplugin_prepPipeline() - a wrapper to the prepPipeline, which does early stage +% +% Usage: +% >> eegplugin_prepPipeline(fig, try_strings, catch_strings); +% +% see also: prepPipeline + +% Author: Kay Robbins, with contributions from Nima Bigdely-Shamlo, Tim Mullen, Christian Kothe, and Cassidy Matousek. + +% This program is free software; you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation; either version 2 of the License, or +% (at your option) any later version. +% +% This program is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with this program; if not, write to the Free Software +% Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +%function eegplugin_clean_rawdata(fig,try_strings,catch_strings) + + +% eegplugin_prepPipeline() - the PREP pipeline plugin +function vers = eegplugin_prepPipeline(fig, trystrs, catchstrs) + +%% Add path to prepPipeline subdirectories if not in the list +tmp = which('getPrepDefaults'); +if isempty(tmp) + myPath = fileparts(which('prepPipeline')); + addpath(genpath(myPath)); +end +vers = getPrepVersion(); + +% create menu +comprep = [trystrs.no_check '[EEG LASTCOM] = pop_prepPipeline(EEG);' catchstrs.new_and_hist]; +menu = findobj(fig, 'tag', 'tools'); +uimenu( menu, 'label', 'Run PREP pipeline', 'callback', comprep, ... + 'separator', 'on'); + diff --git a/PrepPipeline/examples/runESSLevel1ResampleAndDealias.m b/PrepPipeline/examples/runESSLevel1ResampleAndDealias.m deleted file mode 100644 index 6add2f8..0000000 --- a/PrepPipeline/examples/runESSLevel1ResampleAndDealias.m +++ /dev/null @@ -1,30 +0,0 @@ -%% Example using ESS for the BCIT to resample -% ess2Dir = 'D:\TestData\BCITV2\Data\STDL2\ARL_BCIT_CalibrationDriving_Dataset_level_2'; -% ess2File = [ess2Dir filesep 'studyLevel2_description.xml']; -% level2File = 'level2Derived_description.xml'; -% levelDerivedDir = 'D:\TestData\BCITV2\Data\STDL2_256Hz\ARL_BCIT_CalibrationDriving_Dataset_level_2'; - -ess1Dir = 'D:\temp\VEPESS'; -ess1File = [ess1Dir filesep 'study_description.xml']; -level1File = 'level1Derived_description.xml'; -outputDir = 'D:\temp\VEPESSDown'; -mkdir(outputDir); - -%% Check to make sure level 1 study validates -obj1 = level1Study('essFilePath', ess1File); -obj1.validate(); - -%% Get the files from the level1 study -fileNames = getFilename(obj1); -params.resampleOff = false; -params.resampleFrequency = 128; -params.lowPassFrequency = 0; - -for k = 1:length(fileNames) - EEG = pop_loadset(fileNames{k}); - [~, theName, ~] = fileparts(fileNames{k}); - [EEG, resampling] = resampleEEG(EEG, params); - EEG.etc.resampleAndDealias = resampling; - save([outputDir filesep theName '_downSampled.set'], 'EEG', '-v7.3'); -end - diff --git a/PrepPipeline/examples/runESSPrepPipeline.m b/PrepPipeline/examples/runESSPrepPipeline.m deleted file mode 100644 index 3a7a90e..0000000 --- a/PrepPipeline/examples/runESSPrepPipeline.m +++ /dev/null @@ -1,12 +0,0 @@ -%% Example using ESS -ess1Dir = 'D:\TestData\LargData\UCSD_RSVP\Level_1'; -ess2Dir = 'D:\TestData\LargData\UCSD_RSVP\Level_2'; -%% Validate level 1 -ess1File = [ess1Dir filesep 'study_description.xml']; -obj1 = level1Study(ess1File); -obj1.validate(); - -clear obj1; -%% Create a level 2 study -obj2 = level2Study('level1XmlFilePath', ess1File); -obj2.createLevel2Study(ess2Dir); diff --git a/PrepPipeline/examples/runESSResampleAndDealias.m b/PrepPipeline/examples/runESSResampleAndDealias.m deleted file mode 100644 index fdba3fc..0000000 --- a/PrepPipeline/examples/runESSResampleAndDealias.m +++ /dev/null @@ -1,29 +0,0 @@ -%% Example using ESS for the BCIT to resample -ess2Dir = 'D:\TestData\LargData\NCTU\NCTU_DAS\Level2'; -levelDerivedDir = 'D:\TestData\LargData\NCTU\NCTU_DAS\Level2_256Hz'; - -% ess2Dir = 'D:\TestData\BCITV2\Data\STDL2\ARL_BCIT_CalibrationDriving_Dataset_level_2'; -% levelDerivedDir = 'D:\TestData\BCITV2\Data\STDL2_256Hz\ARL_BCIT_CalibrationDriving_Dataset_level_2'; - -% ess2Dir = 'D:\TestData\BCITV2\Data\STDL2\ARL_BCIT_TrafficComplexity_Dataset_level_2'; -% levelDerivedDir = 'D:\TestData\BCITV2\Data\STDL2_256Hz\ARL_BCIT_TrafficComplexity_Dataset_level_2'; - -% ess2Dir = 'D:\TestData\LargData\VEP\ARL_VEP_v1.1.0_Level2'; -% levelDerivedDir = 'D:\TestData\LargData\VEP\ARL_VEP_v1.1.0_Level2_256Hz'; - -% ess2Dir = 'D:\TestData\LargData\UCSD_RSVP\Level_2'; -% levelDerivedDir = 'D:\TestData\LargData\UCSD_RSVP\Level_2_256Hz'; - -%% Check to make sure level 2 study validates -ess2File = [ess2Dir filesep 'studyLevel2_description.xml']; -obj1 = level2Study('level2XmlFilePath', ess2File); -obj1.validate(); - -%% Create a level 2 derived study -obj = levelDerivedStudy('parentStudyXmlFilePath', ess2File); -callbackAndParameters = {@resampleAndDealias, {'resampleOff', false, ... - 'resampleFrequency', 256, 'lowPassFrequency', 100}}; -obj = obj.createLevelDerivedStudy(callbackAndParameters, ... - 'filterDescription', 'Downsample and then low pass to remove alias', ... - 'filterLabel', 'resample', 'levelDerivedFolder', levelDerivedDir); - diff --git a/PrepPipeline/examples/runVEPPrepPipeline.m b/PrepPipeline/examples/runVEPPrepPipeline.m index 76d3c42..7031a2d 100644 --- a/PrepPipeline/examples/runVEPPrepPipeline.m +++ b/PrepPipeline/examples/runVEPPrepPipeline.m @@ -1,50 +1,56 @@ -%% Example: Running the pipeline on a directory of EEG files - -%% Set up the input and the output directories -basename = 'vep'; -indir = 'F:\DataPool\CTADATA\VEP\BiosemiOriginalSetCorrected'; -outdir = 'F:\TempData'; - -%% Make the output directory if needed -if ~exist(outdir, 'dir') - mkdir(outdir) -end - -%% Set up the params structure -params = struct(); -params.lineFrequencies = [60, 120, 180, 212, 240]; -params.referenceChannels = 1:64; -params.evaluationChannels = 1:64; -params.rereferencedChannels = 1:70; -params.detrendChannels = 1:70; -params.lineNoiseChannels = 1:70; - -params.detrendType = 'high pass'; -params.detrendCutoff = 1; -params.referenceType = 'robust'; -params.meanEstimateType = 'median'; -params.interpolationOrder = 'post-reference'; -params.removeInterpolatedChannels = true; -params.keepFiltered = false; -basenameOut = [basename 'robust_1Hz_post_median_unfiltered']; - -%% Get the filelist -fileList = getFileList('FILES', indir); -%% Run the pipeline -for k = 1:length(fileList) - [~, thisName, ~] = fileparts(fileList{k}); - EEG = pop_loadset(fileList{k}); - params.name = thisName; - [EEG, params, computationTimes] = prepPipeline(EEG, params); - fprintf('Computation times (seconds):\n %s\n', ... - getStructureString(computationTimes)); - fprintf('Post-process\n') - EEG = prepPostProcess(EEG, params); - fname = [outdir filesep thisName '.set']; - save(fname, 'EEG', '-mat', '-v7.3'); - if strcmpi(params.errorMsgs, 'verbose') - outputPrepParams(params, 'Prep parameters (non-defaults)'); - outputPrepErrors(EEG.etc.noiseDetection, 'Prep error status'); - end - -end +%% Example: Running the pipeline on a directory of EEG files + +%% Set up the input and the output directories +basename = 'vep'; +% By default this runs on the three example recordings in examples/data and +% writes to examples/output (not tracked by git). Change these to use your own. +exampleDir = fileparts(mfilename('fullpath')); +indir = fullfile(exampleDir, 'data'); % folder of EEGLAB .set files to process +outdir = fullfile(exampleDir, 'output'); % folder for the PREP-processed files + +%% Make the output directory if needed +if ~exist(outdir, 'dir') + mkdir(outdir) +end + +%% Set up the params structure +params = struct(); +params.lineFrequencies = [60, 120, 180, 212, 240]; +params.referenceChannels = 1:64; +params.evaluationChannels = 1:64; +params.rereferencedChannels = 1:70; +params.detrendChannels = 1:70; +params.lineNoiseChannels = 1:70; + +params.detrendType = 'high pass'; +params.detrendCutoff = 1; +params.referenceType = 'robust'; +params.meanEstimateType = 'median'; +params.interpolationOrder = 'post-reference'; +params.removeInterpolatedChannels = true; +params.keepFiltered = false; +basenameOut = [basename 'robust_1Hz_post_median_unfiltered']; + +%% Get the filelist +fileList = getFileList('FILES', indir); +%% Run the pipeline +for k = 1:length(fileList) + [~, thisName, ~] = fileparts(fileList{k}); + EEG = pop_loadset(fileList{k}); + % .fdt files store 32-bit samples, and EEGLAB loads them as single unless + % its double-precision option is set. PREP needs double: convert here. + EEG.data = double(EEG.data); + params.name = thisName; + [EEG, params, computationTimes] = prepPipeline(EEG, params); + fprintf('Computation times (seconds):\n %s\n', ... + getStructureString(computationTimes)); + fprintf('Post-process\n') + EEG = prepPostProcess(EEG, params); + fname = [outdir filesep thisName '.set']; + save(fname, 'EEG', '-mat', '-v7.3'); + if strcmpi(params.errorMsgs, 'verbose') + outputPrepParams(params, 'Prep parameters (non-defaults)'); + outputPrepErrors(EEG.etc.noiseDetection, 'Prep error status'); + end + +end diff --git a/PrepPipeline/examples/runVEPPrepReport.m b/PrepPipeline/examples/runVEPPrepReport.m index 1bdf949..d2fd85b 100644 --- a/PrepPipeline/examples/runVEPPrepReport.m +++ b/PrepPipeline/examples/runVEPPrepReport.m @@ -1,36 +1,42 @@ -%% This script takes a directory of files that have been processed by PREP -% and produces reports. - -%% Read in the file and set the necessary parameters -dataDir = 'F:\TempData'; -summaryFolder = 'F:\TempDataReports'; -publishOn = true; - -%% Get the directory list -inList = dir(dataDir); -inNames = {inList(:).name}; -inTypes = [inList(:).isdir]; -inNames = inNames(~inTypes); - -%% Setup up the names -basename = 'vep'; -summaryReportName = [basename '_summary.html']; -sessionFolder = '.'; -summaryFileName = [summaryFolder filesep summaryReportName]; -if exist(summaryFileName, 'file') - delete(summaryFileName); -end - -%% Publish the reports -for k = 1:length(inNames) - [~, theName, theExt] = fileparts(inNames{k}); - if ~strcmpi(theExt, '.set') && ~strcmpi(theExt, '.mat') - continue; - end - sessionReportName = [theName '.pdf']; - fname = [dataDir filesep inNames{k}]; - load(fname, '-mat'); - sessionFileName = [summaryFolder filesep sessionReportName]; - consoleFID = 1; - publishPrepReport(EEG, summaryFileName, sessionFileName, consoleFID, publishOn); +%% This script takes a directory of files that have been processed by PREP +% and produces reports. + +%% Read in the file and set the necessary parameters +% By default this reports on the output of runVEPPrepPipeline in +% examples/output. Change these to use your own folders. +exampleDir = fileparts(mfilename('fullpath')); +dataDir = fullfile(exampleDir, 'output'); % PREP-processed .set files +summaryFolder = fullfile(exampleDir, 'output', 'reports'); % summary and session reports +if ~exist(summaryFolder, 'dir') + mkdir(summaryFolder); +end +publishOn = true; + +%% Get the directory list +inList = dir(dataDir); +inNames = {inList(:).name}; +inTypes = [inList(:).isdir]; +inNames = inNames(~inTypes); + +%% Setup up the names +basename = 'vep'; +summaryReportName = [basename '_summary.html']; +sessionFolder = '.'; +summaryFileName = [summaryFolder filesep summaryReportName]; +if exist(summaryFileName, 'file') + delete(summaryFileName); +end + +%% Publish the reports +for k = 1:length(inNames) + [~, theName, theExt] = fileparts(inNames{k}); + if ~strcmpi(theExt, '.set') && ~strcmpi(theExt, '.mat') + continue; + end + sessionReportName = [theName '.pdf']; + EEG = pop_loadset('filename', inNames{k}, 'filepath', dataDir); + EEG.data = double(EEG.data); % PREP reporting needs double precision + sessionFileName = [summaryFolder filesep sessionReportName]; + consoleFID = 1; + publishPrepReport(EEG, summaryFileName, sessionFileName, consoleFID, publishOn); end \ No newline at end of file diff --git a/PrepPipeline/interface/MasterGUI.m b/PrepPipeline/interface/MasterGUI.m index 84fffd8..da3c963 100644 --- a/PrepPipeline/interface/MasterGUI.m +++ b/PrepPipeline/interface/MasterGUI.m @@ -1,24 +1,24 @@ -function paramsOut = MasterGUI(hObject, callbackdata, userData, EEG) %#ok -geometry = {1, [1, 1], [1, 1], [1, 1]}; -geomvert = [1,1,1,1]; -title = 'PREP pipeline control panel'; -inputData = struct('signal', EEG, 'name', title, 'userData', userData); -closeOpenWindows(inputData.name); -uilist= {{'style', 'text', 'string', 'Override default parameters for processing step:'}... - {'style', 'pushbutton', 'string', 'Boundary', ... - 'Callback', {@boundaryGUI, inputData}} ... - {'style', 'pushbutton', 'string', 'Reference', ... - 'Callback', {@referenceGUI, inputData}} ... - {'style', 'pushbutton', 'string', 'Detrend', ... - 'Callback', {@detrendGUI, inputData}} ... - {'style', 'pushbutton', 'string', 'Report', ... - 'Callback', {@reportGUI, inputData}} ... - {'style', 'pushbutton', 'string', 'Line noise', ... - 'Callback', {@lineNoiseGUI, inputData}}... - {'style', 'pushbutton', 'string', 'Post process', ... - 'Callback', {@postProcessGUI, inputData}}}; -[~, paramsOut] = inputgui('geometry', geometry, 'geomvert', geomvert, ... - 'uilist', uilist, 'title', title, ... - 'helpcom', 'pophelp(''pop_prepPipeline'')'); - +function paramsOut = MasterGUI(hObject, callbackdata, userData, EEG) %#ok +geometry = {1, [1, 1], [1, 1], [1, 1]}; +geomvert = [1,1,1,1]; +title = 'PREP pipeline control panel'; +inputData = struct('signal', EEG, 'name', title, 'userData', userData); +closeOpenWindows(inputData.name); +uilist= {{'style', 'text', 'string', 'Override default parameters for processing step:'}... + {'style', 'pushbutton', 'string', 'Boundary', ... + 'Callback', {@boundaryGUI, inputData}} ... + {'style', 'pushbutton', 'string', 'Reference', ... + 'Callback', {@referenceGUI, inputData}} ... + {'style', 'pushbutton', 'string', 'Detrend', ... + 'Callback', {@detrendGUI, inputData}} ... + {'style', 'pushbutton', 'string', 'Report', ... + 'Callback', {@reportGUI, inputData}} ... + {'style', 'pushbutton', 'string', 'Line noise', ... + 'Callback', {@lineNoiseGUI, inputData}}... + {'style', 'pushbutton', 'string', 'Post process', ... + 'Callback', {@postProcessGUI, inputData}}}; +[~, paramsOut] = inputgui('geometry', geometry, 'geomvert', geomvert, ... + 'uilist', uilist, 'title', title, ... + 'helpcom', 'pophelp(''pop_prepPipeline'')'); + end % MasterGUI \ No newline at end of file diff --git a/PrepPipeline/interface/displayErrors.m b/PrepPipeline/interface/displayErrors.m index bdc1416..608f99c 100644 --- a/PrepPipeline/interface/displayErrors.m +++ b/PrepPipeline/interface/displayErrors.m @@ -1,26 +1,26 @@ -%% *****************************displayErrors****************************** -%Purpose: -% This returns the errors (if any) that are found when a user enters -% data that does not correspond with the type found by the default -% function. It displays the errors in a pop-up GUI. -%Parameters: -% I errors Cell array of strings; errors found by the -% checkPrepDefaults function -% O loop Integer that either continues the while loop found -% above or exits the loop -%Notes: -% -%Return Value: -% 0 No errors -% 1 Displays errors and continues loop -%************************************************************************** -function displayErrors(errors) - geometry={}; - geomvert=[]; - uilist={}; - for k=1:length(errors) - geometry={geometry{:},1}; - uilist={uilist{:},{'style', 'text', 'string', errors(k)}}; - end - result=inputgui('geometry', geometry, 'geomvert', geomvert, 'uilist', uilist, 'title', 'Reference Errors', 'helpcom', 'pophelp(''pop_eegfiltnew'')'); +%% *****************************displayErrors****************************** +%Purpose: +% This returns the errors (if any) that are found when a user enters +% data that does not correspond with the type found by the default +% function. It displays the errors in a pop-up GUI. +%Parameters: +% I errors Cell array of strings; errors found by the +% checkPrepDefaults function +% O loop Integer that either continues the while loop found +% above or exits the loop +%Notes: +% +%Return Value: +% 0 No errors +% 1 Displays errors and continues loop +%************************************************************************** +function displayErrors(errors) + geometry={}; + geomvert=[]; + uilist={}; + for k=1:length(errors) + geometry={geometry{:},1}; + uilist={uilist{:},{'style', 'text', 'string', errors(k)}}; + end + result=inputgui('geometry', geometry, 'geomvert', geomvert, 'uilist', uilist, 'title', 'Reference Errors', 'helpcom', 'pophelp(''pop_eegfiltnew'')'); end \ No newline at end of file diff --git a/PrepPipeline/pop_prepPipeline.m b/PrepPipeline/pop_prepPipeline.m index d04e857..31193c4 100644 --- a/PrepPipeline/pop_prepPipeline.m +++ b/PrepPipeline/pop_prepPipeline.m @@ -1,153 +1,153 @@ -% pop_prepPipeline() - runs the early stage pipeline to reference and to -% detect bad channels -% -% Usage: -% >> [OUTEEG, com] = pop_prepPipeline(INEEG, params); -% -% Inputs: -% INEEG - input EEG dataset -% params - (optional) structure with parameters to override defaults -% -% Outputs: -% OUTEEG - output dataset -% -% See also: -% prepPipeline, prepPipelineReport, EEGLAB - -% Copyright (C) 2015 Kay Robbins with contributions from Nima -% Bigdely-Shamlo, Christian Kothe, Tim Mullen, and Cassidy Matousek -% -% This program is free software; you can redistribute it and/or modify -% it under the terms of the GNU General Public License as published by -% the Free Software Foundation; either version 2 of the License, or -% (at your option) any later version. -% -% This program is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -% GNU General Public License for more details. -% -% You should have received a copy of the GNU General Public License -% along with this program; if not, write to the Free Software -% Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -function [EEG, com] = pop_prepPipeline(EEG, params) -com = ''; % Return something if user presses the cancel button -if nargin < 1 %% display help if not enough arguments - help pop_prepPipeline; - return; -elseif nargin < 2 - params = struct(); -end - -%% Add path to prepPipeline subdirectories if not in the list -tmp = which('getPrepDefaults'); -if isempty(tmp) - myPath = fileparts(which('prepPipeline')); - addpath(genpath(myPath)); -end - -%% Pop up window -userData = getUserData(); -if nargin < 2 - params = MasterGUI([],[],userData, EEG); -end - -%% Begin the pipeline execution -paramsUpdated = updateParams(params); -com = sprintf('%s = pop_prepPipeline(%s, %s);', inputname(1), ... - struct2str(paramsUpdated)); - -options = getReportOptions(userData, paramsUpdated); - -if strcmpi(options.reportMode, 'normal') || strcmpi(options.reportMode, 'skipReport') - EEG = prepPipeline(EEG, paramsUpdated); -end - -%% Handle reporting -if (strcmpi(options.reportMode, 'normal') || strcmpi(options.reportMode, 'reportOnly')) - publishPrepReport(EEG, options.summaryFilePath, options.sessionFilePath, ... - options.consoleFID, options.publishOn); -end - -%% Perform post-processing -if strcmpi(options.reportMode, 'normal') || strcmpi(options.reportMode, 'skipReport') - EEG = prepPostProcess(EEG, paramsUpdated); -end - - function userData = getUserData() - %% Gets the userData defaults and merges it with the parameters - userData = struct('boundary', [], 'detrend', [], ... - 'lineNoise', [], 'reference', [], ... - 'report', [], 'postProcess', []); - stepNames = fieldnames(userData); - for k = 1:length(stepNames) - defaults = getPrepDefaults(EEG, stepNames{k}); - [theseValues, errors] = checkStructureDefaults(params, ... - defaults); - if ~isempty(errors) - error('pop_prepPipeline:BadParameters', ['|' ... - sprintf('%s|', errors{:})]); - end - userData.(stepNames{k}) = theseValues; - end - end % getUserData - - function paramsOut = updateParams(userDataUpdate) - paramsOut = struct(); - if ~isempty(userDataUpdate) - fNames = fieldnames(userDataUpdate); - for k = 1:length(fNames) - nextStruct = userDataUpdate.(fNames{k}); - nextNames = fieldnames(nextStruct); - for j = 1:length(nextNames) - paramsOut.(nextNames{j}) = nextStruct.(nextNames{j}); - end - end - end - end - - function options = getReportOptions(userData, paramsUpdated) - options = struct('reportMode', '', 'consoleFID', '', 'publishOn', '', ... - 'summaryFilePath', '', 'sessionFilePath', '' ); - options.reportMode = userData.report.reportMode.value; - if isfield(paramsUpdated, 'reportMode') - options.reportMode = paramsUpdated.reportMode; - end - options.consoleFID = userData.report.consoleFID.value; - if isfield(paramsUpdated, 'consoleFID') - options.consoleFID = paramsUpdated.consoleFID; - end - - options.publishOn = userData.report.publishOn.value; - if isfield(paramsUpdated, 'publishOn') - options.publishOn = paramsUpdated.publishOn; - end - - options.summaryFilePath = userData.report.summaryFilePath.value; - if isfield(paramsUpdated, 'summaryFilePath') - options.summaryFilePath = paramsUpdated.summaryFilePath; - end - options.summaryFilePath = resolvePath(options.summaryFilePath); - - options.sessionFilePath = userData.report.sessionFilePath.value; - if isfield(paramsUpdated, 'sessionFilePath') - options.sessionFilePath = paramsUpdated.sessionFilePath; - end - options.sessionFilePath = resolvePath(options.sessionFilePath); - - fprintf('summaryFilePath: %s\n', options.summaryFilePath) - fprintf('sessionFilePath: %s\n', options.sessionFilePath) - end - - function absPath = resolvePath(pathIn) - if isfolder(pathIn) || isfile(pathIn) - % Already exists, maybe absolute - absPath = fullfile(pathIn); - else - % Assume relative to current folder - absPath = fullfile(pwd, pathIn); - end -end - +function [EEG, com] = pop_prepPipeline(EEG, params) +% pop_prepPipeline() - runs the early stage pipeline to reference and to +% detect bad channels +% +% Usage: +% >> [OUTEEG, com] = pop_prepPipeline(INEEG, params); +% +% Inputs: +% INEEG - input EEG dataset +% params - (optional) structure with parameters to override defaults +% +% Outputs: +% OUTEEG - output dataset +% +% See also: +% prepPipeline, prepPipelineReport, EEGLAB + +% Copyright (C) 2015 Kay Robbins with contributions from Nima +% Bigdely-Shamlo, Christian Kothe, Tim Mullen, and Cassidy Matousek +% +% This program is free software; you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation; either version 2 of the License, or +% (at your option) any later version. +% +% This program is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with this program; if not, write to the Free Software +% Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +com = ''; % Return something if user presses the cancel button +if nargin < 1 %% display help if not enough arguments + help pop_prepPipeline; + return; +elseif nargin < 2 + params = struct(); +end + +%% Add path to prepPipeline subdirectories if not in the list +tmp = which('getPrepDefaults'); +if isempty(tmp) + myPath = fileparts(which('prepPipeline')); + addpath(genpath(myPath)); +end + +%% Pop up window +userData = getUserData(); +if nargin < 2 + params = MasterGUI([],[],userData, EEG); +end + +%% Begin the pipeline execution +paramsUpdated = updateParams(params); +com = sprintf('%s = pop_prepPipeline(%s, %s);', inputname(1), ... + struct2str(paramsUpdated)); + +options = getReportOptions(userData, paramsUpdated); + +if strcmpi(options.reportMode, 'normal') || strcmpi(options.reportMode, 'skipReport') + EEG = prepPipeline(EEG, paramsUpdated); +end + +%% Handle reporting +if (strcmpi(options.reportMode, 'normal') || strcmpi(options.reportMode, 'reportOnly')) + publishPrepReport(EEG, options.summaryFilePath, options.sessionFilePath, ... + options.consoleFID, options.publishOn); +end + +%% Perform post-processing +if strcmpi(options.reportMode, 'normal') || strcmpi(options.reportMode, 'skipReport') + EEG = prepPostProcess(EEG, paramsUpdated); +end + + function userData = getUserData() + %% Gets the userData defaults and merges it with the parameters + userData = struct('boundary', [], 'detrend', [], ... + 'lineNoise', [], 'reference', [], ... + 'report', [], 'postProcess', []); + stepNames = fieldnames(userData); + for k = 1:length(stepNames) + defaults = getPrepDefaults(EEG, stepNames{k}); + [theseValues, errors] = checkStructureDefaults(params, ... + defaults); + if ~isempty(errors) + error('pop_prepPipeline:BadParameters', ['|' ... + sprintf('%s|', errors{:})]); + end + userData.(stepNames{k}) = theseValues; + end + end % getUserData + + function paramsOut = updateParams(userDataUpdate) + paramsOut = struct(); + if ~isempty(userDataUpdate) + fNames = fieldnames(userDataUpdate); + for k = 1:length(fNames) + nextStruct = userDataUpdate.(fNames{k}); + nextNames = fieldnames(nextStruct); + for j = 1:length(nextNames) + paramsOut.(nextNames{j}) = nextStruct.(nextNames{j}); + end + end + end + end + + function options = getReportOptions(userData, paramsUpdated) + options = struct('reportMode', '', 'consoleFID', '', 'publishOn', '', ... + 'summaryFilePath', '', 'sessionFilePath', '' ); + options.reportMode = userData.report.reportMode.value; + if isfield(paramsUpdated, 'reportMode') + options.reportMode = paramsUpdated.reportMode; + end + options.consoleFID = userData.report.consoleFID.value; + if isfield(paramsUpdated, 'consoleFID') + options.consoleFID = paramsUpdated.consoleFID; + end + + options.publishOn = userData.report.publishOn.value; + if isfield(paramsUpdated, 'publishOn') + options.publishOn = paramsUpdated.publishOn; + end + + options.summaryFilePath = userData.report.summaryFilePath.value; + if isfield(paramsUpdated, 'summaryFilePath') + options.summaryFilePath = paramsUpdated.summaryFilePath; + end + options.summaryFilePath = resolvePath(options.summaryFilePath); + + options.sessionFilePath = userData.report.sessionFilePath.value; + if isfield(paramsUpdated, 'sessionFilePath') + options.sessionFilePath = paramsUpdated.sessionFilePath; + end + options.sessionFilePath = resolvePath(options.sessionFilePath); + + fprintf('summaryFilePath: %s\n', options.summaryFilePath) + fprintf('sessionFilePath: %s\n', options.sessionFilePath) + end + + function absPath = resolvePath(pathIn) + if isfolder(pathIn) || isfile(pathIn) + % Already exists, maybe absolute + absPath = fullfile(pathIn); + else + % Assume relative to current folder + absPath = fullfile(pwd, pathIn); + end +end + end % pop_prepPipeline \ No newline at end of file diff --git a/PrepPipeline/prepReport.m b/PrepPipeline/prepReport.m index 97bdbb3..309d7ac 100644 --- a/PrepPipeline/prepReport.m +++ b/PrepPipeline/prepReport.m @@ -1,790 +1,790 @@ -function prepReport(EEG, summaryFile, consoleFID, relativeReportLocation) -%% Visualize the EEG output from the PREP processing pipeline. -% -% Calling directly: -% prepReport -% -% This helper reporting script expects that EEG will have an -% EEG.etc.noiseDetection structure containing the report. -% The reporting function appends a summary to the summary report. -% -% Usually the prepReport is called through the function: -% -% publishPrepReport -% - -%% Write data status and report header -reference = struct(); -version = ''; -fullInformation = false; -numbersPerRow = 10; -indent = ' '; - -if isfield(EEG, 'etc') && isfield(EEG.etc, 'noiseDetection') - noiseDetection = EEG.etc.noiseDetection; -else - error('prepReport:NoNoiseDetection', ... - 'PREP reporting relies on EEG.etc.noiseDetection which does not exist'); -end -if isfield(noiseDetection, 'reference') - reference = noiseDetection.reference; -end -if isfield(noiseDetection, 'version') - version = EEG.etc.noiseDetection.version; -end -if isfield(noiseDetection, 'fullReferenceInfo') - fullInformation = EEG.etc.noiseDetection.fullReferenceInfo; -end - -[channels, frames] = size(EEG.data); - -fprintf('Summary file path: %s\n', summaryFile) -fprintf('relative report location: %s\n', relativeReportLocation) -fprintf(consoleFID, '%s\nChannels: %d\nFrames: %d\n', ... - noiseDetection.name, channels, frames); -summaryHeader = [noiseDetection.name '[' ... - num2str(channels) ' channels, ' num2str(frames) ' frames]']; -summaryHeader = [summaryHeader ' Report details']; -writeSummaryHeader(summaryFile, summaryHeader); -originalChannelLabels = noiseDetection.originalChannelLabels; -currentChannelLabels = {EEG.chanlocs.labels}; -[~, iorig, ~] = ... - intersect(originalChannelLabels, currentChannelLabels); -currentChannelsInOriginal = sort(iorig); - -% Write overview status -[errorStatus, errors] = getErrors(noiseDetection); -writeSummaryHeader(summaryFile, errorStatus, 'h4'); -writeHtmlList(summaryFile, errors, 'both'); -fprintf(consoleFID, '%s\n', errorStatus); -writeTextList(consoleFID, errors); - -% Versions -writeSummaryHeader(summaryFile, ['Prep version:' version], 'h4'); -fprintf(consoleFID, 'Prep version: %s\n', version); - -% Events -summaryMsg = ['Data summary: sampling rate ' num2str(EEG.srate) 'Hz']; -writeSummaryHeader(summaryFile, summaryMsg, 'h4'); -fprintf(consoleFID, '%s\n', summaryMsg); -[summary, ~] = reportEvents(consoleFID, EEG); -writeHtmlList(summaryFile, summary, 'both'); - -% Interpolated channels for referencing -if isfield(noiseDetection, 'reference') - writeSummaryHeader(summaryFile, 'Interpolated channels', 'h4'); - removedChannels = getFieldIfExists(noiseDetection, 'removedChannelNumbers'); - interpolatedChannels = getFieldIfExists(noiseDetection, 'interpolatedChannelNumbers'); - stillNoisyChannels = getFieldIfExists(noiseDetection, 'stillNoisyChannelNumbers'); - summaryItem = {['Channels interpolated during reference: [' ... - num2str(interpolatedChannels) ']']; ... - ['Channels still noisy after reference: [' ... - num2str(stillNoisyChannels) ']']; ... - ['Channels removed during post-process: [' ... - num2str(removedChannels) ']' ]}; - writeHtmlList(summaryFile, summaryItem, 'both'); - fprintf(consoleFID, 'Channels interpolated during reference:\n'); - printList(consoleFID, interpolatedChannels, numbersPerRow, indent); - fprintf(consoleFID, 'Channels still noisy after reference:\n'); - printList(consoleFID, stillNoisyChannels, numbersPerRow, indent); - fprintf(consoleFID, 'Channels removed during post-process:\n'); - printList(consoleFID, removedChannels, numbersPerRow, indent); -end - -% Setup visualization parameters - -colors = [0, 0, 0; 0, 1, 0; 1, 0, 0]; -legendStrings = {'Original', 'Before interp' 'Final'}; -symbols = {'+', 'x', 'o'}; -scalpMapInterpolation = 'v4'; -darkElementColor = [0.5, 0.5, 0.5]; -headColor = [0.95, 0.95, 0.95]; -elementColor = [0, 0, 0]; - -%% Line noise removal step -writeSummaryHeader(summaryFile, 'Line noise removal summary', 'h4'); -summary = reportLineNoise(consoleFID, noiseDetection, numbersPerRow, indent); -writeHtmlList(summaryFile, summary, 'both'); - -%% Initial detrend for reference calculation -writeSummaryHeader(summaryFile, 'Detrend summary', 'h4'); -summary = reportDetrend(consoleFID, noiseDetection, numbersPerRow, indent); -writeHtmlList(summaryFile, summary, 'both'); - -%% Spectrum after line noise and detrend -if ~isfield(noiseDetection, 'lineNoise') - fprintf(consoleFID, 'Skipping line noise and detrend\n'); -else - lineChannels = noiseDetection.lineNoise.lineNoiseChannels; - [~, iorig, ~] = intersect(currentChannelsInOriginal, lineChannels); - actualLineChannels = sort(iorig); - channelLabels = {EEG.chanlocs(actualLineChannels).labels}; - tString = noiseDetection.name; - if isfield(noiseDetection, 'detrend') - detrend = noiseDetection.detrend; - detrendChannels = detrend.detrendChannels; - [~, iorig] = intersect(currentChannelsInOriginal, detrendChannels); - isort = sort(iorig); - detrend.detrendChannels = isort(:)'; - EEGNew = removeTrend(EEG, detrend); - else - EEGNew = EEG; - end - [~, ~, badSpectraChannels] = showSpectrum(EEGNew, channelLabels, ... - actualLineChannels, actualLineChannels, tString, 20); - clear EEGNew; - if ~isempty(badSpectraChannels) - badString = ['Channels with no spectra: ' getListString(badSpectraChannels)]; - fprintf(consoleFID, '%s\n', badString); - writeHtmlList(summaryFile, {badString}, 'both'); - end -end - -%% Referencing step -writeSummaryHeader(summaryFile, 'Reference summary', 'h4'); -summary = reportReference(consoleFID, reference, numbersPerRow, indent); -writeHtmlList(summaryFile, summary, 'both'); - -%% Robust channel deviation (referenced) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping robust channel deviation\n'); -else - noisyStatistics = reference.noisyStatistics; - showColorbar = true; - channelInformation = reference.channelInformation; - nosedir = channelInformation.nosedir; - channelLocations = reference.channelLocations; - - [referencedLocations, evaluationChannels, noiseLegendString]= ... - getReportChannelInformation(channelLocations, ... - noisyStatistics.evaluationChannels, noisyStatistics.noisyChannels); - - - interpolatedLocations = getReportChannelInformation(channelLocations, ... - noisyStatistics.evaluationChannels, reference.badChannels); - % Original locations - if ~isfield(reference, 'noisyStatisticsOriginal') || ... - isempty(reference.noisyStatisticsOriginal) - noisyStatisticsOriginal = noisyStatistics; - fprintf(consoleFID, 'No original statistics --- using final for both\n'); - else - noisyStatisticsOriginal = reference.noisyStatisticsOriginal; - end - % Original locations - if ~isfield(reference, 'noisyStatisticsBeforeInterpolation') || ... - isempty(reference.noisyStatisticsBeforeInterpolation) - noisyStatisticsBeforeInterpolation = noisyStatisticsOriginal; - fprintf(consoleFID, ... - 'No statistics before interpolation --- using original for both\n'); - else - noisyStatisticsBeforeInterpolation = ... - reference.noisyStatisticsBeforeInterpolation; - end - originalLocations = getReportChannelInformation(channelLocations, ... - noisyStatistics.evaluationChannels, noisyStatisticsOriginal.noisyChannels); - numberEvaluationChannels = length(evaluationChannels); - - tString = 'Robust channel deviation'; - dataReferenced = noisyStatistics.robustChannelDeviation; - dataReferenced = dataReferenced(evaluationChannels); - dataOriginal = noisyStatisticsOriginal.robustChannelDeviation; - dataOriginal = dataOriginal(evaluationChannels); - dataBeforeInterpolation = ... - noisyStatisticsBeforeInterpolation.robustChannelDeviation; - dataBeforeInterpolation = dataBeforeInterpolation(evaluationChannels); - medRef = noisyStatistics.channelDeviationMedian; - sdnRef = noisyStatistics.channelDeviationSD; - - medOrig = noisyStatisticsOriginal.channelDeviationMedian; - sdnOrig = noisyStatisticsOriginal.channelDeviationSD; - - medInterp = noisyStatisticsBeforeInterpolation.channelDeviationMedian; - sdnInterp = noisyStatisticsBeforeInterpolation.channelDeviationSD; - - scale = max(max(abs(dataOriginal)), max(max(abs(dataBeforeInterpolation)), ... - max(abs(dataReferenced)))); - clim = [-scale, scale]; - fprintf(consoleFID, '\nNoisy channel legend: '); - for j = 1:length(noiseLegendString) - fprintf(consoleFID, '%s\n', noiseLegendString{j}); - end - fprintf(consoleFID, '\n\n'); - plotScalpMap(dataReferenced, referencedLocations, scalpMapInterpolation, ... - showColorbar, headColor, elementColor, clim, nosedir, [tString '(referenced)']) -end - -%% Robust channel deviation (original) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping robust channel deviation (original)\n'); -else - plotScalpMap(dataOriginal, originalLocations, scalpMapInterpolation, ... - showColorbar, headColor, elementColor, clim, nosedir, [tString '(original)']) -end - -%% Robust channel deviation (interpolated) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping robust channel deviation (marking interpolated)\n'); -else - plotScalpMap(dataBeforeInterpolation, interpolatedLocations, scalpMapInterpolation, ... - showColorbar, headColor, elementColor, clim, nosedir, [tString '(marking interpolated)']) -end - -%% Robust deviation window statistics -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping robust deviation window statistics\n'); -else - beforeDeviationLevels = noisyStatisticsOriginal.channelDeviations(evaluationChannels, :); - afterDeviationLevels = noisyStatistics.channelDeviations(evaluationChannels, :); - interpDeviationLevels = ... - noisyStatisticsBeforeInterpolation.channelDeviations(evaluationChannels, :); - beforeDeviation = (beforeDeviationLevels - medOrig)./sdnOrig; - afterDeviation = (afterDeviationLevels - medRef)./sdnRef; - interpDeviation = (interpDeviationLevels - medInterp)./sdnInterp; - medianDeviationsOrig = median(beforeDeviationLevels(:)); - sdDeviationsOrig = mad(beforeDeviationLevels(:), 1)*1.4826; - medianDeviationsRef = median(afterDeviationLevels(:)); - sdDeviationsRef = mad(afterDeviationLevels(:), 1)*1.4826; - thresholdName = 'Deviation score'; - theTitle = {char(noiseDetection.name); char([ thresholdName ' distribution'])}; - showCumulativeDistributions({beforeDeviation(:), interpDeviation(:), afterDeviation(:)}, ... - thresholdName, colors, theTitle, legendStrings, [-5, 5]); - beforeDeviationCounts = ... - sum(beforeDeviation >= noisyStatisticsOriginal.robustDeviationThreshold); - afterDeviationCounts = ... - sum(afterDeviation >= noisyStatistics.robustDeviationThreshold); - interpDeviationCounts = ... - sum(interpDeviation >= noisyStatisticsBeforeInterpolation.robustDeviationThreshold); - beforeTimeScale = (0:length(beforeDeviationCounts)-1)*noisyStatisticsOriginal.correlationWindowSeconds; - afterTimeScale = (0:length(afterDeviationCounts)-1)*noisyStatistics.correlationWindowSeconds; - interpTimeScale = (0:length(interpDeviationCounts)-1)* ... - noisyStatisticsBeforeInterpolation.correlationWindowSeconds; - fractionBefore = mean(beforeDeviationCounts)/numberEvaluationChannels; - fractionAfter = mean(afterDeviationCounts)/numberEvaluationChannels; - counts = {beforeDeviationCounts, interpDeviationCounts, afterDeviationCounts}; - timeScales = {beforeTimeScale, interpTimeScale, afterTimeScale}; - showBadWindows(counts, timeScales, colors, symbols, ... - numberEvaluationChannels, legendStrings, noiseDetection.name, thresholdName); - reports = cell(19, 1); - reports{1} = ['Deviation window statistics (over ' ... - num2str(size(noisyStatistics.channelDeviations, 2)) ' windows)']; - reports{2} = 'Large deviation channel fraction:'; - reports{3} = [indent ' [before=', ... - num2str(fractionBefore) ', after=' num2str(fractionAfter) ']']; - reports{4} = ['Median channel deviation: [before=', ... - num2str(noisyStatisticsOriginal.channelDeviationMedian) ... - ', after=' num2str(noisyStatistics.channelDeviationMedian) ']']; - reports{5} = ['SD channel deviation: [before=', ... - num2str(noisyStatisticsOriginal.channelDeviationSD) ... - ', after=' num2str(noisyStatistics.channelDeviationSD) ']']; - reports{6} = ['Max raw deviation level [before=', ... - num2str(max(beforeDeviationLevels(:))) ', after=' ... - num2str(max(afterDeviationLevels(:))) ']']; - reports{7} = ['Average fraction ' num2str(fractionBefore) ... - ' (' num2str(mean(beforeDeviationCounts)) ' channels)']; - reports{8}= [indent ' not meeting threshold before in each window']; - reports{9} = ['Average fraction ' num2str(fractionAfter) ... - ' (' num2str(mean(afterDeviationCounts)) ' channels)']; - reports{10} = [ indent ' not meeting threshold after in each window']; - quarterChannels = round(length(evaluationChannels)*0.25); - halfChannels = round(length(evaluationChannels)*0.5); - reports{11} = 'Windows with > 1/4 deviation channels:'; - reports{12} = [indent '[before=' ... - num2str(sum(beforeDeviationCounts > quarterChannels)) ... - ', after=' num2str(sum(afterDeviationCounts > quarterChannels)) ']']; - reports{13} = 'Windows with > 1/2 deviation channels:'; - reports{14} = [indent '[before=', ... - num2str(sum(beforeDeviationCounts > halfChannels)) ... - ', after=' num2str(sum(afterDeviationCounts > halfChannels)) ']']; - reports{15} = ['Median window deviations: [before=', ... - num2str(medianDeviationsOrig) ', after=' num2str(medianDeviationsRef) ']']; - reports{16} = ['SD window deviations: [before=', ... - num2str(sdDeviationsOrig) ', after=' num2str(sdDeviationsRef) ']']; - if isfield(noisyStatistics, 'dropOuts') - drops = sum(noisyStatistics.dropOuts, 2)'; - indexDrops = find(drops > 0); - dropList = [indexDrops; drops(indexDrops)]; - if ~isempty(indexDrops) > 0 - reportString = sprintf('%g[%g drops] ', dropList(:)'); - else - reportString = 'None'; - end - reports{17} = ['Channels with dropouts: ' reportString]; - end - fprintf(consoleFID, '%s:\n', reports{1}); - for k = 2:length(reports) - fprintf(consoleFID, '%s\n', reports{k}); - end - writeSummaryHeader(summaryFile, 'Deviation statistics summary', 'h4'); - writeHtmlList(summaryFile, {reports{1}, reports{2}, reports{3}}, 'both'); -end - -%% Median max abs correlation (referenced) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping median max absoluted correlation (referenced)\n'); -else - tString = 'Median max correlation'; - dataReferenced = noisyStatistics.medianMaxCorrelation; - dataReferenced = dataReferenced(evaluationChannels); - clim = [0, 1]; - plotScalpMap(dataReferenced, referencedLocations, scalpMapInterpolation, ... - showColorbar, headColor, elementColor, clim, nosedir, [tString '(referenced)']) -end - -%% Median max abs correlation (original) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping median max abs correlation (original)\n'); -else - tString = 'Median max correlation'; - dataOriginal = noisyStatisticsOriginal.medianMaxCorrelation; - dataOriginal = dataOriginal(evaluationChannels); - clim = [0, 1]; - plotScalpMap(dataOriginal, originalLocations, scalpMapInterpolation, ... - showColorbar, headColor, elementColor, clim, nosedir, [tString '(original)']) -end - -%% Median max abs correlation (interpolated) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, ... - 'Skipping median max abs correlation (marking interpolated)\n'); -else - tString = 'Median max correlation'; - dataBeforeInterpolation = ... - noisyStatisticsBeforeInterpolation.medianMaxCorrelation; - dataBeforeInterpolation = dataBeforeInterpolation(evaluationChannels); - clim = [0, 1]; - plotScalpMap(dataBeforeInterpolation, interpolatedLocations, scalpMapInterpolation, ... - showColorbar, headColor, elementColor, clim, nosedir, [tString '(marking interpolated)']) -end - -%% Mean max abs correlation (referenced) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, ... - 'Skipping median max abs correlation (referenced)\n'); -else - tString = 'Mean max correlation'; - dataReferenced = mean(noisyStatistics.maximumCorrelations, 2); - dataReferenced = dataReferenced(evaluationChannels); - clim = [0, 1]; - plotScalpMap(dataReferenced, referencedLocations, scalpMapInterpolation, ... - showColorbar, headColor, elementColor, clim, nosedir, [tString '(referenced)']) -end - -%% Mean max abs correlation (original) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping mean max abs correlation (original)\n'); -else - tString = 'Mean max correlation'; - dataOriginal = mean(noisyStatisticsOriginal.maximumCorrelations, 2); - dataOriginal = dataOriginal(evaluationChannels); - clim = [0, 1]; - plotScalpMap(dataOriginal, originalLocations, scalpMapInterpolation, ... - showColorbar, headColor, elementColor, clim, nosedir, [tString '(original)']) -end - -%% Mean max abs correlation (interpolated) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, ... - 'Skipping mean max abs correlation (marking interpolated)\n'); -else - tString = 'Mean max correlation'; - dataBeforeInterpolation = ... - mean(noisyStatisticsBeforeInterpolation.maximumCorrelations, 2); - dataBeforeInterpolation = dataBeforeInterpolation(evaluationChannels); - clim = [0, 1]; - plotScalpMap(dataBeforeInterpolation, interpolatedLocations, scalpMapInterpolation, ... - showColorbar, headColor, elementColor, clim, nosedir, [tString '(marking interpolated)']) -end - -%% Bad min max correlation fraction (referenced) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping bad min max correlation (referenced)\n'); -else - tString = 'Min max corr fraction'; - thresholdedCorrelations = noisyStatistics.maximumCorrelations ... - < noisyStatistics.correlationThreshold; - dataReferenced = mean(thresholdedCorrelations, 2); - dataReferenced = dataReferenced(evaluationChannels); - thresholdedCorrelations = noisyStatisticsOriginal.maximumCorrelations ... - < noisyStatisticsOriginal.correlationThreshold; - dataOriginal = mean(thresholdedCorrelations, 2); - dataOriginal = dataOriginal(evaluationChannels); - thresholdedCorrelations = noisyStatisticsBeforeInterpolation.maximumCorrelations ... - < noisyStatisticsBeforeInterpolation.correlationThreshold; - dataBeforeInterpolation = mean(thresholdedCorrelations, 2); - dataBeforeInterpolation = dataBeforeInterpolation(evaluationChannels); - clim = [0, 2*reference.badTimeThreshold]; - plotScalpMap(dataReferenced, referencedLocations, scalpMapInterpolation, ... - showColorbar, headColor, darkElementColor, clim, nosedir, [tString '(referenced)']) -end -%% Bad min max correlation fraction(original) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping median max abs correlation (original)\n'); -else - plotScalpMap(dataOriginal, originalLocations, scalpMapInterpolation, ... - showColorbar, headColor, darkElementColor, clim, nosedir, [tString '(original)']) -end - -%% Bad min max correlation fraction (interpolated) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, ... - 'Skipping bad min max correlation fraction (marking interpolated)\n'); -else - plotScalpMap(dataBeforeInterpolation, interpolatedLocations, scalpMapInterpolation, ... - showColorbar, headColor, darkElementColor, clim, nosedir, [tString '(marking interpolated)']) -end - -%% Correlation window statistics -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping correlation window statistics\n'); -else - beforeCorrelationLevels = ... - noisyStatisticsOriginal.maximumCorrelations(evaluationChannels, :); - afterCorrelationLevels = ... - noisyStatistics.maximumCorrelations(evaluationChannels, :); - interpCorrelationLevels = ... - noisyStatisticsBeforeInterpolation.maximumCorrelations(evaluationChannels, :); - thresholdName = 'Maximum correlation'; - theTitle = {char(noiseDetection.name); char([thresholdName ' distribution'])}; - showCumulativeDistributions( ... - {beforeCorrelationLevels(:),interpCorrelationLevels(:), afterCorrelationLevels(:)}, ... - thresholdName, colors, theTitle, legendStrings, [0, 1]); - beforeCorrelationCounts = sum(beforeCorrelationLevels <= ... - noisyStatisticsOriginal.correlationThreshold); - afterCorrelationCounts = sum(afterCorrelationLevels <= ... - noisyStatistics.correlationThreshold); - interpCorrelationCounts = sum(interpCorrelationLevels <= ... - noisyStatisticsBeforeInterpolation.correlationThreshold); - beforeTimeScale = (0:length(beforeCorrelationCounts)-1)* ... - noisyStatisticsOriginal.correlationWindowSeconds; - afterTimeScale = (0:length(afterCorrelationCounts)-1)* ... - noisyStatistics.correlationWindowSeconds; - interpTimeScale = (0:length(interpCorrelationCounts)-1)* ... - noisyStatisticsBeforeInterpolation.correlationWindowSeconds; - counts = {beforeCorrelationCounts, interpCorrelationCounts, afterCorrelationCounts}; - timeScales = {beforeTimeScale, interpTimeScale, afterTimeScale}; - showBadWindows(counts, timeScales, colors, symbols, ... - numberEvaluationChannels, legendStrings, noiseDetection.name, thresholdName); - fractionBefore = mean(beforeCorrelationCounts)/numberEvaluationChannels; - fractionAfter = mean(afterCorrelationCounts)/numberEvaluationChannels; - reports = cell(10, 1); - reports{1} = ['Max correlation window statistics (over ' ... - num2str(size(noisyStatistics.maximumCorrelations, 2)) ' windows)']; - reports{2} = ['Overall median maximum correlation [before=', ... - num2str(median(noisyStatisticsOriginal.medianMaxCorrelation(:))) ... - ', after=' num2str(median(noisyStatistics.medianMaxCorrelation(:))) ']']; - reports{3} = ['Low max correlation fraction [before=', ... - num2str(fractionBefore) ', after=' num2str(fractionAfter) ']']; - reports{4} = ['Minimum max correlation level [before=', ... - num2str(min(beforeCorrelationLevels(:))) ', after=' ... - num2str(min(afterCorrelationLevels(:))) ']']; - reports{5} = ['Average fraction ' num2str(fractionBefore) ... - ' (' num2str(mean(beforeCorrelationCounts)) ' channels):']; - reports{6} = [indent ' not meeting threshold before in each window']; - reports{7} = ['Average fraction ' num2str(fractionAfter) ... - ' (' num2str(mean(afterCorrelationCounts)) ' channels):']; - reports{8} = [indent ' not meeting threshold after in each window']; - quarterChannels = round(length(evaluationChannels)*0.25); - halfChannels = round(length(evaluationChannels)*0.5); - reports{9} = ['Windows with > 1/4 bad channels: [before=', ... - num2str(sum(beforeCorrelationCounts > quarterChannels)) ... - ', after=' num2str(sum(afterCorrelationCounts > quarterChannels)) ']']; - reports{10} = ['Windows with > 1/2 bad channels: [before=', ... - num2str(sum(beforeCorrelationCounts > halfChannels)) ... - ', after=' num2str(sum(afterCorrelationCounts > halfChannels)) ']']; - fprintf(consoleFID, '%s:\n', reports{1}); - for k = 2:length(reports) - fprintf(consoleFID, '%s\n', reports{k}); - end - writeSummaryHeader(summaryFile, 'Correlation statistics summary', 'h4'); - writeHtmlList(summaryFile, {reports{1}, reports{2}}, 'both'); -end - -%% Bad ransac fraction (referenced) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping bad ransac fraction (referenced)\n'); -else - tString = 'Ransac fraction failed'; - dataReferenced = noisyStatistics.ransacBadWindowFraction; - dataReferenced = dataReferenced(evaluationChannels); - clim = [0, 1]; - plotScalpMap(dataReferenced, referencedLocations, scalpMapInterpolation, ... - showColorbar, headColor, darkElementColor, clim, nosedir, [tString '(referenced)']) -end - -%% Bad ransac fraction (original) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping bad ransac fraction (original)\n'); -else - dataOriginal = noisyStatisticsOriginal.ransacBadWindowFraction; - dataOriginal = dataOriginal(evaluationChannels); - plotScalpMap(dataOriginal, originalLocations, scalpMapInterpolation, ... - showColorbar, headColor, darkElementColor, clim, nosedir, [tString '(original)']) -end - -%% Bad ransac fraction (interpolated) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping bad ransac fraction (marking interpolated)\n'); -else - dataBeforeInterpolation = noisyStatisticsBeforeInterpolation.ransacBadWindowFraction; - dataBeforeInterpolation = dataBeforeInterpolation(evaluationChannels); - plotScalpMap(dataBeforeInterpolation, interpolatedLocations, scalpMapInterpolation, ... - showColorbar, headColor, darkElementColor, clim, nosedir, [tString '(marking interpolated)']) -end - -%% Channels with poor ransac correlations -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping channels with poor ransac correlations\n'); -else - beforeRansacLevels = ... - noisyStatisticsOriginal.ransacCorrelations(evaluationChannels, :); - afterRansacLevels = ... - noisyStatistics.ransacCorrelations(evaluationChannels, :); - interpRansacLevels = ... - noisyStatisticsBeforeInterpolation.ransacCorrelations(evaluationChannels, :); - thresholdName = 'Ransac correlation'; - theTitle = {char([noiseDetection.name ': ' thresholdName ' distribution'])}; - showCumulativeDistributions({beforeRansacLevels(:), ... - interpRansacLevels(:), afterRansacLevels(:)}, ... - thresholdName, colors, theTitle, legendStrings, [0, 1]); - - beforeRansacCounts = sum(beforeRansacLevels <= ... - noisyStatisticsOriginal.ransacCorrelationThreshold); - afterRansacCounts = sum(afterRansacLevels <= ... - noisyStatistics.ransacCorrelationThreshold); - interpRansacCounts = sum(interpRansacLevels <= ... - noisyStatisticsBeforeInterpolation.ransacCorrelationThreshold); - beforeTimeScale = (0:length(beforeRansacCounts)-1)* ... - noisyStatisticsOriginal.ransacWindowSeconds; - afterTimeScale = (0:length(afterRansacCounts)-1)* ... - noisyStatistics.ransacWindowSeconds; - interpTimeScale = (0:length(interpRansacCounts)-1)* ... - noisyStatisticsBeforeInterpolation.ransacWindowSeconds; - counts = {beforeRansacCounts, interpRansacCounts, afterRansacCounts}; - timeScales = {beforeTimeScale, interpTimeScale, afterTimeScale}; - showBadWindows(counts, timeScales, colors, symbols, ... - numberEvaluationChannels, legendStrings, noiseDetection.name, thresholdName); - - fractionBefore = mean(beforeRansacCounts)/numberEvaluationChannels; - fractionAfter = mean(afterRansacCounts)/numberEvaluationChannels; - reports = cell(9, 0); - reports{1} = ['Ransac window statistics (over ' ... - num2str(size(afterRansacLevels, 2)) ' windows)']; - reports{2} = ['Low ransac channel fraction [before=', ... - num2str(fractionBefore) ', after=' num2str(fractionAfter) ']']; - reports{3} = ['Minimum ransac correlation [before=', ... - num2str(min(beforeRansacLevels(:))) ', after=' ... - num2str(min(afterRansacLevels(:))) ']']; - reports{4} = ['Average fraction ' num2str(fractionBefore) ... - ' (' num2str(mean(beforeRansacCounts)) ' channels):']; - reports{5} = [indent ' not meeting threshold before in each window']; - reports{6} = ['Average fraction ' num2str(fractionAfter) ... - ' (' num2str(mean(afterRansacCounts)) ' channels):']; - reports{7} = [indent ' not meeting threshold after in each window']; - quarterChannels = round(length(evaluationChannels)*0.25); - halfChannels = round(length(evaluationChannels)*0.5); - reports{8} = ['Windows with > 1/4 bad ransac channels: [before=', ... - num2str(sum(beforeRansacCounts > quarterChannels)) ... - ', after=' num2str(sum(afterRansacCounts > quarterChannels)) ']']; - reports{9} = ['Windows with > 1/2 bad ransac channels: [before=', ... - num2str(sum(beforeRansacCounts > halfChannels)) ... - ', after=' num2str(sum(afterRansacCounts > halfChannels)) ']']; - fprintf(consoleFID, '%s:\n', reports{1}); - for k = 2:length(reports) - fprintf(consoleFID, '%s\n', reports{k}); - end - writeSummaryHeader(summaryFile, 'Ransac statistics summary', 'h4'); - writeHtmlList(summaryFile, {reports{1}, reports{2}}, 'both'); -end -%% HF noise Z-score (referenced) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping HF noise Z-score (referenced)\n'); -else - tString = 'Z-score HF SNR'; - dataReferenced = noisyStatistics.zscoreHFNoise; - dataReferenced = dataReferenced(evaluationChannels); - dataOriginal = noisyStatisticsOriginal.zscoreHFNoise; - dataOriginal = dataOriginal(evaluationChannels); - dataBeforeInterpolation = noisyStatisticsBeforeInterpolation.zscoreHFNoise; - dataBeforeInterpolation = dataBeforeInterpolation(evaluationChannels); - scale = max(max(abs(dataOriginal)), max(max(abs(dataReferenced)), ... - max(abs(dataBeforeInterpolation)))); - % scale = max(max(abs(dataOriginal), abs(dataReferenced))); - clim = [-scale, scale]; - plotScalpMap(dataReferenced, referencedLocations, scalpMapInterpolation, ... - showColorbar, headColor, elementColor, clim, nosedir, [tString '(referenced)']) -end - - -%% HF noise Z-score (original) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping HF noise Z-score (original)\n'); -else - plotScalpMap(dataOriginal, originalLocations, scalpMapInterpolation, ... - showColorbar, headColor, elementColor, clim, nosedir, [tString '(original)']) -end - -%% HF noise Z-score (interpolated) -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping HF noise Z-score (marking interpolated)\n'); -else - plotScalpMap(dataBeforeInterpolation, interpolatedLocations, scalpMapInterpolation, ... - showColorbar, headColor, elementColor, clim, nosedir, [tString '(marking interpolated)']) -end - -%% HF noise window stats -if isempty(reference) || ~fullInformation - fprintf(consoleFID, 'Skipping HF window stats\n'); -else - beforeNoiseLevels = noisyStatisticsOriginal.noiseLevels(evaluationChannels, :); - afterNoiseLevels = noisyStatistics.noiseLevels(evaluationChannels, :); - interpNoiseLevels = ... - noisyStatisticsBeforeInterpolation.noiseLevels(evaluationChannels, :); - medianNoiseOrig = median(beforeNoiseLevels(:)); - sdNoiseOrig = mad(beforeNoiseLevels(:), 1)*1.4826; - medianNoiseRef = median(afterNoiseLevels(:)); - sdNoiseRef = mad(afterNoiseLevels(:), 1)*1.4826; - medianNoiseInterp = median(interpNoiseLevels(:)); - sdNoiseInterp = mad(interpNoiseLevels(:), 1)*1.4826; - beforeNoise = (beforeNoiseLevels - medianNoiseOrig)./sdNoiseOrig; - afterNoise = (afterNoiseLevels - medianNoiseRef)./sdNoiseRef; - interpNoise = (interpNoiseLevels - medianNoiseInterp)./sdNoiseInterp; - thresholdName = 'HF noise'; - theTitle = {char(noiseDetection.name); [thresholdName ' HF noise distribution']}; - showCumulativeDistributions({beforeNoise(:), interpNoise(:), afterNoise(:)}, ... - thresholdName, colors, theTitle, legendStrings, [-5, 5]); - beforeNoiseCounts = sum(beforeNoise >= ... - noisyStatisticsOriginal.highFrequencyNoiseThreshold); - afterNoiseCounts = sum(afterNoise >= ... - noisyStatistics.highFrequencyNoiseThreshold); - interpNoiseCounts = sum(interpNoise >= ... - noisyStatisticsBeforeInterpolation.highFrequencyNoiseThreshold); - beforeTimeScale = (0:length(beforeNoiseCounts)-1)* ... - noisyStatisticsOriginal.correlationWindowSeconds; - afterTimeScale = (0:length(afterNoiseCounts)-1)* ... - noisyStatistics.correlationWindowSeconds; - interpTimeScale = (0:length(interpNoiseCounts)-1)* ... - noisyStatisticsBeforeInterpolation.correlationWindowSeconds; - counts = {beforeNoiseCounts, interpNoiseCounts, afterNoiseCounts}; - timeScales = {beforeTimeScale, interpTimeScale, afterTimeScale}; - showBadWindows(counts, timeScales, colors, symbols, ... - numberEvaluationChannels, legendStrings, noiseDetection.name, thresholdName); - - fractionBefore = mean(beforeNoiseCounts)/numberEvaluationChannels; - fractionAfter = mean(afterNoiseCounts)/numberEvaluationChannels; - reports = cell(17,0); - reports{1} = ['Noise window statistics (over ' ... - num2str(size(noisyStatistics.noiseLevels, 2)) ' windows)']; - reports{2} = 'Channel fraction with HF noise:'; - reports{3} = [indent '[before=', ... - num2str(fractionBefore) ', after=' num2str(fractionAfter) ']']; - reports{4} = ['Median noisiness: [before=', ... - num2str(noisyStatisticsOriginal.noisinessMedian) ... - ', after=' num2str(noisyStatistics.noisinessMedian) ']']; - reports{5} = ['SD noisiness: [before=', ... - num2str(noisyStatisticsOriginal.noisinessSD) ... - ', after=' num2str(noisyStatistics.noisinessSD) ']']; - reports{6} = ['Max HF noise levels [before=', ... - num2str(max(beforeNoiseLevels(:))) ', after=' ... - num2str(max(afterNoiseLevels(:))) ']']; - reports{7} = ['Average fraction ' num2str(fractionBefore) ... - ' (' num2str(mean(beforeNoiseCounts)) ' channels):']; - reports{8} = [indent ' not meeting threshold before in each window']; - reports{9} = ['Average fraction ' num2str(fractionAfter) ... - ' (' num2str(mean(afterNoiseCounts)) ' channels):']; - reports{10} = [indent ' not meeting threshold after in each window']; - reports{11} = [indent ' not meeting threshold after relative to before in each window']; - quarterChannels = round(length(evaluationChannels)*0.25); - halfChannels = round(length(evaluationChannels)*0.5); - reports{12} = 'Windows with > 1/4 HF channels:'; - reports{13} = [indent '[before=', ... - num2str(sum(beforeNoiseCounts > quarterChannels)) ... - ', after=' num2str(sum(afterNoiseCounts > quarterChannels)) ']']; - reports{14} = 'Windows with > 1/2 HF channels:'; - reports{15} = [indent '[before=', ... - num2str(sum(beforeNoiseCounts > halfChannels)) ... - ', after=' num2str(sum(afterNoiseCounts > halfChannels)) ']']; - reports{16} = ['Median window HF: [before=', ... - num2str(medianNoiseOrig) ', after=' num2str(medianNoiseRef) ']']; - reports{17} = ['SD window HF: [before=', ... - num2str(sdNoiseOrig) ', after=' num2str(sdNoiseRef) ']']; - fprintf(consoleFID, '%s:\n', reports{1}); - for k = 2:length(reports) - fprintf(consoleFID, '%s\n', reports{k}); - end - writeSummaryHeader(summaryFile, 'HF statistics summary', 'h4'); - writeHtmlList(summaryFile, {reports{1}, reports{2}, reports{3}}, 'both'); -end - - -%% Noisy average vs robust average reference -if isempty(reference) || ~fullInformation || ... - ~isfield(reference, 'referenceSignal') || isempty(reference.referenceSignal) - fprintf(consoleFID, 'Skipping noisy vs robust average reference\n'); -else - corrAverage = corr(reference.referenceSignal(:), ... - reference.referenceSignalOriginal(:)); - tString = { noiseDetection.name, ... - ['Comparison of reference signals (corr=' num2str(corrAverage) ')']}; - figure('Name', tString{2}) - plot(reference.referenceSignal, reference.referenceSignalOriginal, '.k'); - xlabel('Robust average reference') - ylabel('Ordinary average reference'); - title(tString, 'Interpreter', 'None'); - corrString = ['Ordinary vs robust average reference (unfiltered) correlation: ' ... - num2str(corrAverage)]; - writeSummaryHeader(summaryFile, corrString, 'h4'); -end - -%% Noisy and robust average reference by time -if isempty(reference) || ~fullInformation || ... - ~isfield(reference, 'referenceSignal') || isempty(reference.referenceSignal) - fprintf(consoleFID, 'Skipping noisy and robust average reference by time\n'); -else - tString = { noiseDetection.name, 'ordinary - robust average reference signals'}; - t = (0:length(reference.referenceSignal) - 1)/EEG.srate; - figure('Name', tString{2}) - plot(t, reference.referenceSignalOriginal - reference.referenceSignal, '.k'); - xlabel('Seconds') - ylabel('Original - robust'); - title(tString, 'Interpreter', 'None'); -end - -%% Noisy vs robust average reference (filtered) -if isempty(reference) || ~fullInformation || ... - ~isfield(reference, 'referenceSignal') || isempty(reference.referenceSignal) - fprintf(consoleFID, 'Skipping noisy vs robust average reference (filtered)\n'); -else - EEGTemp = eeg_emptyset(); - EEGTemp.nbchan = 2; - a = reference.referenceSignal; - b = reference.referenceSignalOriginal; - EEGTemp.pnts = length(a); - EEGTemp.data = [a(:)'; b(:)']; - EEGTemp.srate = EEG.srate; - EEGTemp = pop_eegfiltnew(EEGTemp, noiseDetection.detrend.detrendCutoff, []); - corrAverage = corr(EEGTemp.data(1, :)', EEGTemp.data(2, :)'); - tString = { noiseDetection.name, ... - ['Comparison of reference signals (corr=' num2str(corrAverage) ')']}; - figure('Name', tString{2}) - plot(EEGTemp.data(1, :), EEGTemp.data(2, :), '.k'); - xlabel('Robust average reference') - ylabel('Ordinary average reference'); - title(tString, 'Interpreter', 'None'); - corrString = ['Ordinary vs robust average reference (filtered) correlation: ' ... - num2str(corrAverage)]; - writeSummaryHeader(summaryFile, corrString, 'h4'); -end -%% Noisy minus robust average reference by time -if isempty(reference) || ~fullInformation || ... - ~isfield(reference, 'referenceSignal') || isempty(reference.referenceSignal) - fprintf(consoleFID, 'Skipping noisy minus robust average reference by time\n'); -else - tString = { noiseDetection.name, 'ordinary - robust average reference signals'}; - t = (0:length(EEGTemp.data(2, :)) - 1)/EEG.srate; - figure('Name', tString{2}) - plot(t, EEGTemp.data(2, :) - EEGTemp.data(1, :), '.k'); - xlabel('Seconds') - ylabel('Average - robust'); - title(tString, 'Interpreter', 'None'); -end +function prepReport(EEG, summaryFile, consoleFID, relativeReportLocation) +%% Visualize the EEG output from the PREP processing pipeline. +% +% Calling directly: +% prepReport +% +% This helper reporting script expects that EEG will have an +% EEG.etc.noiseDetection structure containing the report. +% The reporting function appends a summary to the summary report. +% +% Usually the prepReport is called through the function: +% +% publishPrepReport +% + +%% Write data status and report header +reference = struct(); +version = ''; +fullInformation = false; +numbersPerRow = 10; +indent = ' '; + +if isfield(EEG, 'etc') && isfield(EEG.etc, 'noiseDetection') + noiseDetection = EEG.etc.noiseDetection; +else + error('prepReport:NoNoiseDetection', ... + 'PREP reporting relies on EEG.etc.noiseDetection which does not exist'); +end +if isfield(noiseDetection, 'reference') + reference = noiseDetection.reference; +end +if isfield(noiseDetection, 'version') + version = EEG.etc.noiseDetection.version; +end +if isfield(noiseDetection, 'fullReferenceInfo') + fullInformation = EEG.etc.noiseDetection.fullReferenceInfo; +end + +[channels, frames] = size(EEG.data); + +fprintf('Summary file path: %s\n', summaryFile) +fprintf('relative report location: %s\n', relativeReportLocation) +fprintf(consoleFID, '%s\nChannels: %d\nFrames: %d\n', ... + noiseDetection.name, channels, frames); +summaryHeader = [noiseDetection.name '[' ... + num2str(channels) ' channels, ' num2str(frames) ' frames]']; +summaryHeader = [summaryHeader ' Report details']; +writeSummaryHeader(summaryFile, summaryHeader); +originalChannelLabels = noiseDetection.originalChannelLabels; +currentChannelLabels = {EEG.chanlocs.labels}; +[~, iorig, ~] = ... + intersect(originalChannelLabels, currentChannelLabels); +currentChannelsInOriginal = sort(iorig); + +% Write overview status +[errorStatus, errors] = getErrors(noiseDetection); +writeSummaryHeader(summaryFile, errorStatus, 'h4'); +writeHtmlList(summaryFile, errors, 'both'); +fprintf(consoleFID, '%s\n', errorStatus); +writeTextList(consoleFID, errors); + +% Versions +writeSummaryHeader(summaryFile, ['Prep version:' version], 'h4'); +fprintf(consoleFID, 'Prep version: %s\n', version); + +% Events +summaryMsg = ['Data summary: sampling rate ' num2str(EEG.srate) 'Hz']; +writeSummaryHeader(summaryFile, summaryMsg, 'h4'); +fprintf(consoleFID, '%s\n', summaryMsg); +[summary, ~] = reportEvents(consoleFID, EEG); +writeHtmlList(summaryFile, summary, 'both'); + +% Interpolated channels for referencing +if isfield(noiseDetection, 'reference') + writeSummaryHeader(summaryFile, 'Interpolated channels', 'h4'); + removedChannels = getFieldIfExists(noiseDetection, 'removedChannelNumbers'); + interpolatedChannels = getFieldIfExists(noiseDetection, 'interpolatedChannelNumbers'); + stillNoisyChannels = getFieldIfExists(noiseDetection, 'stillNoisyChannelNumbers'); + summaryItem = {['Channels interpolated during reference: [' ... + num2str(interpolatedChannels) ']']; ... + ['Channels still noisy after reference: [' ... + num2str(stillNoisyChannels) ']']; ... + ['Channels removed during post-process: [' ... + num2str(removedChannels) ']' ]}; + writeHtmlList(summaryFile, summaryItem, 'both'); + fprintf(consoleFID, 'Channels interpolated during reference:\n'); + printList(consoleFID, interpolatedChannels, numbersPerRow, indent); + fprintf(consoleFID, 'Channels still noisy after reference:\n'); + printList(consoleFID, stillNoisyChannels, numbersPerRow, indent); + fprintf(consoleFID, 'Channels removed during post-process:\n'); + printList(consoleFID, removedChannels, numbersPerRow, indent); +end + +% Setup visualization parameters + +colors = [0, 0, 0; 0, 1, 0; 1, 0, 0]; +legendStrings = {'Original', 'Before interp' 'Final'}; +symbols = {'+', 'x', 'o'}; +scalpMapInterpolation = 'v4'; +darkElementColor = [0.5, 0.5, 0.5]; +headColor = [0.95, 0.95, 0.95]; +elementColor = [0, 0, 0]; + +%% Line noise removal step +writeSummaryHeader(summaryFile, 'Line noise removal summary', 'h4'); +summary = reportLineNoise(consoleFID, noiseDetection, numbersPerRow, indent); +writeHtmlList(summaryFile, summary, 'both'); + +%% Initial detrend for reference calculation +writeSummaryHeader(summaryFile, 'Detrend summary', 'h4'); +summary = reportDetrend(consoleFID, noiseDetection, numbersPerRow, indent); +writeHtmlList(summaryFile, summary, 'both'); + +%% Spectrum after line noise and detrend +if ~isfield(noiseDetection, 'lineNoise') + fprintf(consoleFID, 'Skipping line noise and detrend\n'); +else + lineChannels = noiseDetection.lineNoise.lineNoiseChannels; + [~, iorig, ~] = intersect(currentChannelsInOriginal, lineChannels); + actualLineChannels = sort(iorig); + channelLabels = {EEG.chanlocs(actualLineChannels).labels}; + tString = noiseDetection.name; + if isfield(noiseDetection, 'detrend') + detrend = noiseDetection.detrend; + detrendChannels = detrend.detrendChannels; + [~, iorig] = intersect(currentChannelsInOriginal, detrendChannels); + isort = sort(iorig); + detrend.detrendChannels = isort(:)'; + EEGNew = removeTrend(EEG, detrend); + else + EEGNew = EEG; + end + [~, ~, badSpectraChannels] = showSpectrum(EEGNew, channelLabels, ... + actualLineChannels, actualLineChannels, tString, 20); + clear EEGNew; + if ~isempty(badSpectraChannels) + badString = ['Channels with no spectra: ' getListString(badSpectraChannels)]; + fprintf(consoleFID, '%s\n', badString); + writeHtmlList(summaryFile, {badString}, 'both'); + end +end + +%% Referencing step +writeSummaryHeader(summaryFile, 'Reference summary', 'h4'); +summary = reportReference(consoleFID, reference, numbersPerRow, indent); +writeHtmlList(summaryFile, summary, 'both'); + +%% Robust channel deviation (referenced) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping robust channel deviation\n'); +else + noisyStatistics = reference.noisyStatistics; + showColorbar = true; + channelInformation = reference.channelInformation; + nosedir = channelInformation.nosedir; + channelLocations = reference.channelLocations; + + [referencedLocations, evaluationChannels, noiseLegendString]= ... + getReportChannelInformation(channelLocations, ... + noisyStatistics.evaluationChannels, noisyStatistics.noisyChannels); + + + interpolatedLocations = getReportChannelInformation(channelLocations, ... + noisyStatistics.evaluationChannels, reference.badChannels); + % Original locations + if ~isfield(reference, 'noisyStatisticsOriginal') || ... + isempty(reference.noisyStatisticsOriginal) + noisyStatisticsOriginal = noisyStatistics; + fprintf(consoleFID, 'No original statistics --- using final for both\n'); + else + noisyStatisticsOriginal = reference.noisyStatisticsOriginal; + end + % Original locations + if ~isfield(reference, 'noisyStatisticsBeforeInterpolation') || ... + isempty(reference.noisyStatisticsBeforeInterpolation) + noisyStatisticsBeforeInterpolation = noisyStatisticsOriginal; + fprintf(consoleFID, ... + 'No statistics before interpolation --- using original for both\n'); + else + noisyStatisticsBeforeInterpolation = ... + reference.noisyStatisticsBeforeInterpolation; + end + originalLocations = getReportChannelInformation(channelLocations, ... + noisyStatistics.evaluationChannels, noisyStatisticsOriginal.noisyChannels); + numberEvaluationChannels = length(evaluationChannels); + + tString = 'Robust channel deviation'; + dataReferenced = noisyStatistics.robustChannelDeviation; + dataReferenced = dataReferenced(evaluationChannels); + dataOriginal = noisyStatisticsOriginal.robustChannelDeviation; + dataOriginal = dataOriginal(evaluationChannels); + dataBeforeInterpolation = ... + noisyStatisticsBeforeInterpolation.robustChannelDeviation; + dataBeforeInterpolation = dataBeforeInterpolation(evaluationChannels); + medRef = noisyStatistics.channelDeviationMedian; + sdnRef = noisyStatistics.channelDeviationSD; + + medOrig = noisyStatisticsOriginal.channelDeviationMedian; + sdnOrig = noisyStatisticsOriginal.channelDeviationSD; + + medInterp = noisyStatisticsBeforeInterpolation.channelDeviationMedian; + sdnInterp = noisyStatisticsBeforeInterpolation.channelDeviationSD; + + scale = max(max(abs(dataOriginal)), max(max(abs(dataBeforeInterpolation)), ... + max(abs(dataReferenced)))); + clim = [-scale, scale]; + fprintf(consoleFID, '\nNoisy channel legend: '); + for j = 1:length(noiseLegendString) + fprintf(consoleFID, '%s\n', noiseLegendString{j}); + end + fprintf(consoleFID, '\n\n'); + plotScalpMap(dataReferenced, referencedLocations, scalpMapInterpolation, ... + showColorbar, headColor, elementColor, clim, nosedir, [tString '(referenced)']) +end + +%% Robust channel deviation (original) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping robust channel deviation (original)\n'); +else + plotScalpMap(dataOriginal, originalLocations, scalpMapInterpolation, ... + showColorbar, headColor, elementColor, clim, nosedir, [tString '(original)']) +end + +%% Robust channel deviation (interpolated) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping robust channel deviation (marking interpolated)\n'); +else + plotScalpMap(dataBeforeInterpolation, interpolatedLocations, scalpMapInterpolation, ... + showColorbar, headColor, elementColor, clim, nosedir, [tString '(marking interpolated)']) +end + +%% Robust deviation window statistics +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping robust deviation window statistics\n'); +else + beforeDeviationLevels = noisyStatisticsOriginal.channelDeviations(evaluationChannels, :); + afterDeviationLevels = noisyStatistics.channelDeviations(evaluationChannels, :); + interpDeviationLevels = ... + noisyStatisticsBeforeInterpolation.channelDeviations(evaluationChannels, :); + beforeDeviation = (beforeDeviationLevels - medOrig)./sdnOrig; + afterDeviation = (afterDeviationLevels - medRef)./sdnRef; + interpDeviation = (interpDeviationLevels - medInterp)./sdnInterp; + medianDeviationsOrig = median(beforeDeviationLevels(:)); + sdDeviationsOrig = mad(beforeDeviationLevels(:), 1)*1.4826; + medianDeviationsRef = median(afterDeviationLevels(:)); + sdDeviationsRef = mad(afterDeviationLevels(:), 1)*1.4826; + thresholdName = 'Deviation score'; + theTitle = {char(noiseDetection.name); char([ thresholdName ' distribution'])}; + showCumulativeDistributions({beforeDeviation(:), interpDeviation(:), afterDeviation(:)}, ... + thresholdName, colors, theTitle, legendStrings, [-5, 5]); + beforeDeviationCounts = ... + sum(beforeDeviation >= noisyStatisticsOriginal.robustDeviationThreshold); + afterDeviationCounts = ... + sum(afterDeviation >= noisyStatistics.robustDeviationThreshold); + interpDeviationCounts = ... + sum(interpDeviation >= noisyStatisticsBeforeInterpolation.robustDeviationThreshold); + beforeTimeScale = (0:length(beforeDeviationCounts)-1)*noisyStatisticsOriginal.correlationWindowSeconds; + afterTimeScale = (0:length(afterDeviationCounts)-1)*noisyStatistics.correlationWindowSeconds; + interpTimeScale = (0:length(interpDeviationCounts)-1)* ... + noisyStatisticsBeforeInterpolation.correlationWindowSeconds; + fractionBefore = mean(beforeDeviationCounts)/numberEvaluationChannels; + fractionAfter = mean(afterDeviationCounts)/numberEvaluationChannels; + counts = {beforeDeviationCounts, interpDeviationCounts, afterDeviationCounts}; + timeScales = {beforeTimeScale, interpTimeScale, afterTimeScale}; + showBadWindows(counts, timeScales, colors, symbols, ... + numberEvaluationChannels, legendStrings, noiseDetection.name, thresholdName); + reports = cell(19, 1); + reports{1} = ['Deviation window statistics (over ' ... + num2str(size(noisyStatistics.channelDeviations, 2)) ' windows)']; + reports{2} = 'Large deviation channel fraction:'; + reports{3} = [indent ' [before=', ... + num2str(fractionBefore) ', after=' num2str(fractionAfter) ']']; + reports{4} = ['Median channel deviation: [before=', ... + num2str(noisyStatisticsOriginal.channelDeviationMedian) ... + ', after=' num2str(noisyStatistics.channelDeviationMedian) ']']; + reports{5} = ['SD channel deviation: [before=', ... + num2str(noisyStatisticsOriginal.channelDeviationSD) ... + ', after=' num2str(noisyStatistics.channelDeviationSD) ']']; + reports{6} = ['Max raw deviation level [before=', ... + num2str(max(beforeDeviationLevels(:))) ', after=' ... + num2str(max(afterDeviationLevels(:))) ']']; + reports{7} = ['Average fraction ' num2str(fractionBefore) ... + ' (' num2str(mean(beforeDeviationCounts)) ' channels)']; + reports{8}= [indent ' not meeting threshold before in each window']; + reports{9} = ['Average fraction ' num2str(fractionAfter) ... + ' (' num2str(mean(afterDeviationCounts)) ' channels)']; + reports{10} = [ indent ' not meeting threshold after in each window']; + quarterChannels = round(length(evaluationChannels)*0.25); + halfChannels = round(length(evaluationChannels)*0.5); + reports{11} = 'Windows with > 1/4 deviation channels:'; + reports{12} = [indent '[before=' ... + num2str(sum(beforeDeviationCounts > quarterChannels)) ... + ', after=' num2str(sum(afterDeviationCounts > quarterChannels)) ']']; + reports{13} = 'Windows with > 1/2 deviation channels:'; + reports{14} = [indent '[before=', ... + num2str(sum(beforeDeviationCounts > halfChannels)) ... + ', after=' num2str(sum(afterDeviationCounts > halfChannels)) ']']; + reports{15} = ['Median window deviations: [before=', ... + num2str(medianDeviationsOrig) ', after=' num2str(medianDeviationsRef) ']']; + reports{16} = ['SD window deviations: [before=', ... + num2str(sdDeviationsOrig) ', after=' num2str(sdDeviationsRef) ']']; + if isfield(noisyStatistics, 'dropOuts') + drops = sum(noisyStatistics.dropOuts, 2)'; + indexDrops = find(drops > 0); + dropList = [indexDrops; drops(indexDrops)]; + if ~isempty(indexDrops) > 0 + reportString = sprintf('%g[%g drops] ', dropList(:)'); + else + reportString = 'None'; + end + reports{17} = ['Channels with dropouts: ' reportString]; + end + fprintf(consoleFID, '%s:\n', reports{1}); + for k = 2:length(reports) + fprintf(consoleFID, '%s\n', reports{k}); + end + writeSummaryHeader(summaryFile, 'Deviation statistics summary', 'h4'); + writeHtmlList(summaryFile, {reports{1}, reports{2}, reports{3}}, 'both'); +end + +%% Median max abs correlation (referenced) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping median max absoluted correlation (referenced)\n'); +else + tString = 'Median max correlation'; + dataReferenced = noisyStatistics.medianMaxCorrelation; + dataReferenced = dataReferenced(evaluationChannels); + clim = [0, 1]; + plotScalpMap(dataReferenced, referencedLocations, scalpMapInterpolation, ... + showColorbar, headColor, elementColor, clim, nosedir, [tString '(referenced)']) +end + +%% Median max abs correlation (original) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping median max abs correlation (original)\n'); +else + tString = 'Median max correlation'; + dataOriginal = noisyStatisticsOriginal.medianMaxCorrelation; + dataOriginal = dataOriginal(evaluationChannels); + clim = [0, 1]; + plotScalpMap(dataOriginal, originalLocations, scalpMapInterpolation, ... + showColorbar, headColor, elementColor, clim, nosedir, [tString '(original)']) +end + +%% Median max abs correlation (interpolated) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, ... + 'Skipping median max abs correlation (marking interpolated)\n'); +else + tString = 'Median max correlation'; + dataBeforeInterpolation = ... + noisyStatisticsBeforeInterpolation.medianMaxCorrelation; + dataBeforeInterpolation = dataBeforeInterpolation(evaluationChannels); + clim = [0, 1]; + plotScalpMap(dataBeforeInterpolation, interpolatedLocations, scalpMapInterpolation, ... + showColorbar, headColor, elementColor, clim, nosedir, [tString '(marking interpolated)']) +end + +%% Mean max abs correlation (referenced) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, ... + 'Skipping median max abs correlation (referenced)\n'); +else + tString = 'Mean max correlation'; + dataReferenced = mean(noisyStatistics.maximumCorrelations, 2); + dataReferenced = dataReferenced(evaluationChannels); + clim = [0, 1]; + plotScalpMap(dataReferenced, referencedLocations, scalpMapInterpolation, ... + showColorbar, headColor, elementColor, clim, nosedir, [tString '(referenced)']) +end + +%% Mean max abs correlation (original) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping mean max abs correlation (original)\n'); +else + tString = 'Mean max correlation'; + dataOriginal = mean(noisyStatisticsOriginal.maximumCorrelations, 2); + dataOriginal = dataOriginal(evaluationChannels); + clim = [0, 1]; + plotScalpMap(dataOriginal, originalLocations, scalpMapInterpolation, ... + showColorbar, headColor, elementColor, clim, nosedir, [tString '(original)']) +end + +%% Mean max abs correlation (interpolated) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, ... + 'Skipping mean max abs correlation (marking interpolated)\n'); +else + tString = 'Mean max correlation'; + dataBeforeInterpolation = ... + mean(noisyStatisticsBeforeInterpolation.maximumCorrelations, 2); + dataBeforeInterpolation = dataBeforeInterpolation(evaluationChannels); + clim = [0, 1]; + plotScalpMap(dataBeforeInterpolation, interpolatedLocations, scalpMapInterpolation, ... + showColorbar, headColor, elementColor, clim, nosedir, [tString '(marking interpolated)']) +end + +%% Bad min max correlation fraction (referenced) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping bad min max correlation (referenced)\n'); +else + tString = 'Min max corr fraction'; + thresholdedCorrelations = noisyStatistics.maximumCorrelations ... + < noisyStatistics.correlationThreshold; + dataReferenced = mean(thresholdedCorrelations, 2); + dataReferenced = dataReferenced(evaluationChannels); + thresholdedCorrelations = noisyStatisticsOriginal.maximumCorrelations ... + < noisyStatisticsOriginal.correlationThreshold; + dataOriginal = mean(thresholdedCorrelations, 2); + dataOriginal = dataOriginal(evaluationChannels); + thresholdedCorrelations = noisyStatisticsBeforeInterpolation.maximumCorrelations ... + < noisyStatisticsBeforeInterpolation.correlationThreshold; + dataBeforeInterpolation = mean(thresholdedCorrelations, 2); + dataBeforeInterpolation = dataBeforeInterpolation(evaluationChannels); + clim = [0, 2*reference.badTimeThreshold]; + plotScalpMap(dataReferenced, referencedLocations, scalpMapInterpolation, ... + showColorbar, headColor, darkElementColor, clim, nosedir, [tString '(referenced)']) +end +%% Bad min max correlation fraction(original) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping median max abs correlation (original)\n'); +else + plotScalpMap(dataOriginal, originalLocations, scalpMapInterpolation, ... + showColorbar, headColor, darkElementColor, clim, nosedir, [tString '(original)']) +end + +%% Bad min max correlation fraction (interpolated) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, ... + 'Skipping bad min max correlation fraction (marking interpolated)\n'); +else + plotScalpMap(dataBeforeInterpolation, interpolatedLocations, scalpMapInterpolation, ... + showColorbar, headColor, darkElementColor, clim, nosedir, [tString '(marking interpolated)']) +end + +%% Correlation window statistics +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping correlation window statistics\n'); +else + beforeCorrelationLevels = ... + noisyStatisticsOriginal.maximumCorrelations(evaluationChannels, :); + afterCorrelationLevels = ... + noisyStatistics.maximumCorrelations(evaluationChannels, :); + interpCorrelationLevels = ... + noisyStatisticsBeforeInterpolation.maximumCorrelations(evaluationChannels, :); + thresholdName = 'Maximum correlation'; + theTitle = {char(noiseDetection.name); char([thresholdName ' distribution'])}; + showCumulativeDistributions( ... + {beforeCorrelationLevels(:),interpCorrelationLevels(:), afterCorrelationLevels(:)}, ... + thresholdName, colors, theTitle, legendStrings, [0, 1]); + beforeCorrelationCounts = sum(beforeCorrelationLevels <= ... + noisyStatisticsOriginal.correlationThreshold); + afterCorrelationCounts = sum(afterCorrelationLevels <= ... + noisyStatistics.correlationThreshold); + interpCorrelationCounts = sum(interpCorrelationLevels <= ... + noisyStatisticsBeforeInterpolation.correlationThreshold); + beforeTimeScale = (0:length(beforeCorrelationCounts)-1)* ... + noisyStatisticsOriginal.correlationWindowSeconds; + afterTimeScale = (0:length(afterCorrelationCounts)-1)* ... + noisyStatistics.correlationWindowSeconds; + interpTimeScale = (0:length(interpCorrelationCounts)-1)* ... + noisyStatisticsBeforeInterpolation.correlationWindowSeconds; + counts = {beforeCorrelationCounts, interpCorrelationCounts, afterCorrelationCounts}; + timeScales = {beforeTimeScale, interpTimeScale, afterTimeScale}; + showBadWindows(counts, timeScales, colors, symbols, ... + numberEvaluationChannels, legendStrings, noiseDetection.name, thresholdName); + fractionBefore = mean(beforeCorrelationCounts)/numberEvaluationChannels; + fractionAfter = mean(afterCorrelationCounts)/numberEvaluationChannels; + reports = cell(10, 1); + reports{1} = ['Max correlation window statistics (over ' ... + num2str(size(noisyStatistics.maximumCorrelations, 2)) ' windows)']; + reports{2} = ['Overall median maximum correlation [before=', ... + num2str(median(noisyStatisticsOriginal.medianMaxCorrelation(:))) ... + ', after=' num2str(median(noisyStatistics.medianMaxCorrelation(:))) ']']; + reports{3} = ['Low max correlation fraction [before=', ... + num2str(fractionBefore) ', after=' num2str(fractionAfter) ']']; + reports{4} = ['Minimum max correlation level [before=', ... + num2str(min(beforeCorrelationLevels(:))) ', after=' ... + num2str(min(afterCorrelationLevels(:))) ']']; + reports{5} = ['Average fraction ' num2str(fractionBefore) ... + ' (' num2str(mean(beforeCorrelationCounts)) ' channels):']; + reports{6} = [indent ' not meeting threshold before in each window']; + reports{7} = ['Average fraction ' num2str(fractionAfter) ... + ' (' num2str(mean(afterCorrelationCounts)) ' channels):']; + reports{8} = [indent ' not meeting threshold after in each window']; + quarterChannels = round(length(evaluationChannels)*0.25); + halfChannels = round(length(evaluationChannels)*0.5); + reports{9} = ['Windows with > 1/4 bad channels: [before=', ... + num2str(sum(beforeCorrelationCounts > quarterChannels)) ... + ', after=' num2str(sum(afterCorrelationCounts > quarterChannels)) ']']; + reports{10} = ['Windows with > 1/2 bad channels: [before=', ... + num2str(sum(beforeCorrelationCounts > halfChannels)) ... + ', after=' num2str(sum(afterCorrelationCounts > halfChannels)) ']']; + fprintf(consoleFID, '%s:\n', reports{1}); + for k = 2:length(reports) + fprintf(consoleFID, '%s\n', reports{k}); + end + writeSummaryHeader(summaryFile, 'Correlation statistics summary', 'h4'); + writeHtmlList(summaryFile, {reports{1}, reports{2}}, 'both'); +end + +%% Bad ransac fraction (referenced) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping bad ransac fraction (referenced)\n'); +else + tString = 'Ransac fraction failed'; + dataReferenced = noisyStatistics.ransacBadWindowFraction; + dataReferenced = dataReferenced(evaluationChannels); + clim = [0, 1]; + plotScalpMap(dataReferenced, referencedLocations, scalpMapInterpolation, ... + showColorbar, headColor, darkElementColor, clim, nosedir, [tString '(referenced)']) +end + +%% Bad ransac fraction (original) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping bad ransac fraction (original)\n'); +else + dataOriginal = noisyStatisticsOriginal.ransacBadWindowFraction; + dataOriginal = dataOriginal(evaluationChannels); + plotScalpMap(dataOriginal, originalLocations, scalpMapInterpolation, ... + showColorbar, headColor, darkElementColor, clim, nosedir, [tString '(original)']) +end + +%% Bad ransac fraction (interpolated) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping bad ransac fraction (marking interpolated)\n'); +else + dataBeforeInterpolation = noisyStatisticsBeforeInterpolation.ransacBadWindowFraction; + dataBeforeInterpolation = dataBeforeInterpolation(evaluationChannels); + plotScalpMap(dataBeforeInterpolation, interpolatedLocations, scalpMapInterpolation, ... + showColorbar, headColor, darkElementColor, clim, nosedir, [tString '(marking interpolated)']) +end + +%% Channels with poor ransac correlations +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping channels with poor ransac correlations\n'); +else + beforeRansacLevels = ... + noisyStatisticsOriginal.ransacCorrelations(evaluationChannels, :); + afterRansacLevels = ... + noisyStatistics.ransacCorrelations(evaluationChannels, :); + interpRansacLevels = ... + noisyStatisticsBeforeInterpolation.ransacCorrelations(evaluationChannels, :); + thresholdName = 'Ransac correlation'; + theTitle = {char([noiseDetection.name ': ' thresholdName ' distribution'])}; + showCumulativeDistributions({beforeRansacLevels(:), ... + interpRansacLevels(:), afterRansacLevels(:)}, ... + thresholdName, colors, theTitle, legendStrings, [0, 1]); + + beforeRansacCounts = sum(beforeRansacLevels <= ... + noisyStatisticsOriginal.ransacCorrelationThreshold); + afterRansacCounts = sum(afterRansacLevels <= ... + noisyStatistics.ransacCorrelationThreshold); + interpRansacCounts = sum(interpRansacLevels <= ... + noisyStatisticsBeforeInterpolation.ransacCorrelationThreshold); + beforeTimeScale = (0:length(beforeRansacCounts)-1)* ... + noisyStatisticsOriginal.ransacWindowSeconds; + afterTimeScale = (0:length(afterRansacCounts)-1)* ... + noisyStatistics.ransacWindowSeconds; + interpTimeScale = (0:length(interpRansacCounts)-1)* ... + noisyStatisticsBeforeInterpolation.ransacWindowSeconds; + counts = {beforeRansacCounts, interpRansacCounts, afterRansacCounts}; + timeScales = {beforeTimeScale, interpTimeScale, afterTimeScale}; + showBadWindows(counts, timeScales, colors, symbols, ... + numberEvaluationChannels, legendStrings, noiseDetection.name, thresholdName); + + fractionBefore = mean(beforeRansacCounts)/numberEvaluationChannels; + fractionAfter = mean(afterRansacCounts)/numberEvaluationChannels; + reports = cell(9, 0); + reports{1} = ['Ransac window statistics (over ' ... + num2str(size(afterRansacLevels, 2)) ' windows)']; + reports{2} = ['Low ransac channel fraction [before=', ... + num2str(fractionBefore) ', after=' num2str(fractionAfter) ']']; + reports{3} = ['Minimum ransac correlation [before=', ... + num2str(min(beforeRansacLevels(:))) ', after=' ... + num2str(min(afterRansacLevels(:))) ']']; + reports{4} = ['Average fraction ' num2str(fractionBefore) ... + ' (' num2str(mean(beforeRansacCounts)) ' channels):']; + reports{5} = [indent ' not meeting threshold before in each window']; + reports{6} = ['Average fraction ' num2str(fractionAfter) ... + ' (' num2str(mean(afterRansacCounts)) ' channels):']; + reports{7} = [indent ' not meeting threshold after in each window']; + quarterChannels = round(length(evaluationChannels)*0.25); + halfChannels = round(length(evaluationChannels)*0.5); + reports{8} = ['Windows with > 1/4 bad ransac channels: [before=', ... + num2str(sum(beforeRansacCounts > quarterChannels)) ... + ', after=' num2str(sum(afterRansacCounts > quarterChannels)) ']']; + reports{9} = ['Windows with > 1/2 bad ransac channels: [before=', ... + num2str(sum(beforeRansacCounts > halfChannels)) ... + ', after=' num2str(sum(afterRansacCounts > halfChannels)) ']']; + fprintf(consoleFID, '%s:\n', reports{1}); + for k = 2:length(reports) + fprintf(consoleFID, '%s\n', reports{k}); + end + writeSummaryHeader(summaryFile, 'Ransac statistics summary', 'h4'); + writeHtmlList(summaryFile, {reports{1}, reports{2}}, 'both'); +end +%% HF noise Z-score (referenced) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping HF noise Z-score (referenced)\n'); +else + tString = 'Z-score HF SNR'; + dataReferenced = noisyStatistics.zscoreHFNoise; + dataReferenced = dataReferenced(evaluationChannels); + dataOriginal = noisyStatisticsOriginal.zscoreHFNoise; + dataOriginal = dataOriginal(evaluationChannels); + dataBeforeInterpolation = noisyStatisticsBeforeInterpolation.zscoreHFNoise; + dataBeforeInterpolation = dataBeforeInterpolation(evaluationChannels); + scale = max(max(abs(dataOriginal)), max(max(abs(dataReferenced)), ... + max(abs(dataBeforeInterpolation)))); + % scale = max(max(abs(dataOriginal), abs(dataReferenced))); + clim = [-scale, scale]; + plotScalpMap(dataReferenced, referencedLocations, scalpMapInterpolation, ... + showColorbar, headColor, elementColor, clim, nosedir, [tString '(referenced)']) +end + + +%% HF noise Z-score (original) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping HF noise Z-score (original)\n'); +else + plotScalpMap(dataOriginal, originalLocations, scalpMapInterpolation, ... + showColorbar, headColor, elementColor, clim, nosedir, [tString '(original)']) +end + +%% HF noise Z-score (interpolated) +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping HF noise Z-score (marking interpolated)\n'); +else + plotScalpMap(dataBeforeInterpolation, interpolatedLocations, scalpMapInterpolation, ... + showColorbar, headColor, elementColor, clim, nosedir, [tString '(marking interpolated)']) +end + +%% HF noise window stats +if isempty(reference) || ~fullInformation + fprintf(consoleFID, 'Skipping HF window stats\n'); +else + beforeNoiseLevels = noisyStatisticsOriginal.noiseLevels(evaluationChannels, :); + afterNoiseLevels = noisyStatistics.noiseLevels(evaluationChannels, :); + interpNoiseLevels = ... + noisyStatisticsBeforeInterpolation.noiseLevels(evaluationChannels, :); + medianNoiseOrig = median(beforeNoiseLevels(:)); + sdNoiseOrig = mad(beforeNoiseLevels(:), 1)*1.4826; + medianNoiseRef = median(afterNoiseLevels(:)); + sdNoiseRef = mad(afterNoiseLevels(:), 1)*1.4826; + medianNoiseInterp = median(interpNoiseLevels(:)); + sdNoiseInterp = mad(interpNoiseLevels(:), 1)*1.4826; + beforeNoise = (beforeNoiseLevels - medianNoiseOrig)./sdNoiseOrig; + afterNoise = (afterNoiseLevels - medianNoiseRef)./sdNoiseRef; + interpNoise = (interpNoiseLevels - medianNoiseInterp)./sdNoiseInterp; + thresholdName = 'HF noise'; + theTitle = {char(noiseDetection.name); [thresholdName ' HF noise distribution']}; + showCumulativeDistributions({beforeNoise(:), interpNoise(:), afterNoise(:)}, ... + thresholdName, colors, theTitle, legendStrings, [-5, 5]); + beforeNoiseCounts = sum(beforeNoise >= ... + noisyStatisticsOriginal.highFrequencyNoiseThreshold); + afterNoiseCounts = sum(afterNoise >= ... + noisyStatistics.highFrequencyNoiseThreshold); + interpNoiseCounts = sum(interpNoise >= ... + noisyStatisticsBeforeInterpolation.highFrequencyNoiseThreshold); + beforeTimeScale = (0:length(beforeNoiseCounts)-1)* ... + noisyStatisticsOriginal.correlationWindowSeconds; + afterTimeScale = (0:length(afterNoiseCounts)-1)* ... + noisyStatistics.correlationWindowSeconds; + interpTimeScale = (0:length(interpNoiseCounts)-1)* ... + noisyStatisticsBeforeInterpolation.correlationWindowSeconds; + counts = {beforeNoiseCounts, interpNoiseCounts, afterNoiseCounts}; + timeScales = {beforeTimeScale, interpTimeScale, afterTimeScale}; + showBadWindows(counts, timeScales, colors, symbols, ... + numberEvaluationChannels, legendStrings, noiseDetection.name, thresholdName); + + fractionBefore = mean(beforeNoiseCounts)/numberEvaluationChannels; + fractionAfter = mean(afterNoiseCounts)/numberEvaluationChannels; + reports = cell(17,0); + reports{1} = ['Noise window statistics (over ' ... + num2str(size(noisyStatistics.noiseLevels, 2)) ' windows)']; + reports{2} = 'Channel fraction with HF noise:'; + reports{3} = [indent '[before=', ... + num2str(fractionBefore) ', after=' num2str(fractionAfter) ']']; + reports{4} = ['Median noisiness: [before=', ... + num2str(noisyStatisticsOriginal.noisinessMedian) ... + ', after=' num2str(noisyStatistics.noisinessMedian) ']']; + reports{5} = ['SD noisiness: [before=', ... + num2str(noisyStatisticsOriginal.noisinessSD) ... + ', after=' num2str(noisyStatistics.noisinessSD) ']']; + reports{6} = ['Max HF noise levels [before=', ... + num2str(max(beforeNoiseLevels(:))) ', after=' ... + num2str(max(afterNoiseLevels(:))) ']']; + reports{7} = ['Average fraction ' num2str(fractionBefore) ... + ' (' num2str(mean(beforeNoiseCounts)) ' channels):']; + reports{8} = [indent ' not meeting threshold before in each window']; + reports{9} = ['Average fraction ' num2str(fractionAfter) ... + ' (' num2str(mean(afterNoiseCounts)) ' channels):']; + reports{10} = [indent ' not meeting threshold after in each window']; + reports{11} = [indent ' not meeting threshold after relative to before in each window']; + quarterChannels = round(length(evaluationChannels)*0.25); + halfChannels = round(length(evaluationChannels)*0.5); + reports{12} = 'Windows with > 1/4 HF channels:'; + reports{13} = [indent '[before=', ... + num2str(sum(beforeNoiseCounts > quarterChannels)) ... + ', after=' num2str(sum(afterNoiseCounts > quarterChannels)) ']']; + reports{14} = 'Windows with > 1/2 HF channels:'; + reports{15} = [indent '[before=', ... + num2str(sum(beforeNoiseCounts > halfChannels)) ... + ', after=' num2str(sum(afterNoiseCounts > halfChannels)) ']']; + reports{16} = ['Median window HF: [before=', ... + num2str(medianNoiseOrig) ', after=' num2str(medianNoiseRef) ']']; + reports{17} = ['SD window HF: [before=', ... + num2str(sdNoiseOrig) ', after=' num2str(sdNoiseRef) ']']; + fprintf(consoleFID, '%s:\n', reports{1}); + for k = 2:length(reports) + fprintf(consoleFID, '%s\n', reports{k}); + end + writeSummaryHeader(summaryFile, 'HF statistics summary', 'h4'); + writeHtmlList(summaryFile, {reports{1}, reports{2}, reports{3}}, 'both'); +end + + +%% Noisy average vs robust average reference +if isempty(reference) || ~fullInformation || ... + ~isfield(reference, 'referenceSignal') || isempty(reference.referenceSignal) + fprintf(consoleFID, 'Skipping noisy vs robust average reference\n'); +else + corrAverage = corr(reference.referenceSignal(:), ... + reference.referenceSignalOriginal(:)); + tString = { noiseDetection.name, ... + ['Comparison of reference signals (corr=' num2str(corrAverage) ')']}; + figure('Name', tString{2}) + plot(reference.referenceSignal, reference.referenceSignalOriginal, '.k'); + xlabel('Robust average reference') + ylabel('Ordinary average reference'); + title(tString, 'Interpreter', 'None'); + corrString = ['Ordinary vs robust average reference (unfiltered) correlation: ' ... + num2str(corrAverage)]; + writeSummaryHeader(summaryFile, corrString, 'h4'); +end + +%% Noisy and robust average reference by time +if isempty(reference) || ~fullInformation || ... + ~isfield(reference, 'referenceSignal') || isempty(reference.referenceSignal) + fprintf(consoleFID, 'Skipping noisy and robust average reference by time\n'); +else + tString = { noiseDetection.name, 'ordinary - robust average reference signals'}; + t = (0:length(reference.referenceSignal) - 1)/EEG.srate; + figure('Name', tString{2}) + plot(t, reference.referenceSignalOriginal - reference.referenceSignal, '.k'); + xlabel('Seconds') + ylabel('Original - robust'); + title(tString, 'Interpreter', 'None'); +end + +%% Noisy vs robust average reference (filtered) +if isempty(reference) || ~fullInformation || ... + ~isfield(reference, 'referenceSignal') || isempty(reference.referenceSignal) + fprintf(consoleFID, 'Skipping noisy vs robust average reference (filtered)\n'); +else + EEGTemp = eeg_emptyset(); + EEGTemp.nbchan = 2; + a = reference.referenceSignal; + b = reference.referenceSignalOriginal; + EEGTemp.pnts = length(a); + EEGTemp.data = [a(:)'; b(:)']; + EEGTemp.srate = EEG.srate; + EEGTemp = pop_eegfiltnew(EEGTemp, noiseDetection.detrend.detrendCutoff, []); + corrAverage = corr(EEGTemp.data(1, :)', EEGTemp.data(2, :)'); + tString = { noiseDetection.name, ... + ['Comparison of reference signals (corr=' num2str(corrAverage) ')']}; + figure('Name', tString{2}) + plot(EEGTemp.data(1, :), EEGTemp.data(2, :), '.k'); + xlabel('Robust average reference') + ylabel('Ordinary average reference'); + title(tString, 'Interpreter', 'None'); + corrString = ['Ordinary vs robust average reference (filtered) correlation: ' ... + num2str(corrAverage)]; + writeSummaryHeader(summaryFile, corrString, 'h4'); +end +%% Noisy minus robust average reference by time +if isempty(reference) || ~fullInformation || ... + ~isfield(reference, 'referenceSignal') || isempty(reference.referenceSignal) + fprintf(consoleFID, 'Skipping noisy minus robust average reference by time\n'); +else + tString = { noiseDetection.name, 'ordinary - robust average reference signals'}; + t = (0:length(EEGTemp.data(2, :)) - 1)/EEG.srate; + figure('Name', tString{2}) + plot(t, EEGTemp.data(2, :) - EEGTemp.data(1, :), '.k'); + xlabel('Seconds') + ylabel('Average - robust'); + title(tString, 'Interpreter', 'None'); +end diff --git a/PrepPipeline/preplicense.txt b/PrepPipeline/preplicense.txt index 2234892..412e512 100644 --- a/PrepPipeline/preplicense.txt +++ b/PrepPipeline/preplicense.txt @@ -1,236 +1,236 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - ADDITIONAL NOTE - -The PREP Pipeline is designed and distributed for research purposes only and -should not be used for medical purposes. The authors accept no responsibility -for its use in this manner. + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + ADDITIONAL NOTE + +The PREP Pipeline is designed and distributed for research purposes only and +should not be used for medical purposes. The authors accept no responsibility +for its use in this manner. diff --git a/PrepPipeline/publishPrepReport.m b/PrepPipeline/publishPrepReport.m index 70db1c5..2e6ef3b 100644 --- a/PrepPipeline/publishPrepReport.m +++ b/PrepPipeline/publishPrepReport.m @@ -1,134 +1,134 @@ -function [] = publishPrepReport(EEG, summaryFilePath, sessionFilePath, ... - consoleFID, publishOn) -% Create a published report from the PREP pipeline. -% -% Note: In addition to creating a report for the EEG, it appends a -% summary of the file to an existing summary file. This enables the -% function to be called successfully on a collection and creates a summary -% of the collection. -% -% Parameters: -% EEG EEGLAB structure with the EEG.etc.noiseDetection -% structure created by the PREP pipeline -% summaryFilePath File name including path of the summary file -% sessionFilePath File name including path of the individual report -% consoleID Open file descriptor for echoing output (usually 1 -% indication the Command Window). -% publishOn If true (default), report is published and -% figures are closed. If false, output and figures -% are displayed in the normal way. The figures -% are not closed. This option is useful when -% you want to manipulate the figures in some way. -% -% Output: -% If the publish option is on, this function will create a report -% for the EEG and will append a summary to a specified summary file. -% If the publish option is off, the function will just run the -% prepReport script. -% -% Author: Kay Robbins, UTSA, March 2015. -% -% -%% Handle the parameters - if (nargin < 4) - error('publishPrepReport:NotEnoughParameters', ... - ['Usage: publishPrepReport(EEG, summaryFilePath, ' ... - 'sessionFilePath, consoleFID, publishOn)']); - elseif nargin < 5 || isempty(publishOn) - publishOn = true; - end - -%% Make sure that EEGLAB is working in double precision - [backupOptionsFile, currentOptionsFile, warningsState] = setupForEEGLAB(); - finishup = onCleanup(@() cleanup(backupOptionsFile, currentOptionsFile, ... - warningsState)); - -%% Setup up files and assign variables needed for publish in base workspace - wrapperScriptName = 'prepReportWrapper'; - [summaryFolder, summaryName, summaryExt] = fileparts(summaryFilePath); - [sessionFolder, sessionName, sessionExt] = fileparts(sessionFilePath); - summaryReportLocation = fullfile(summaryFolder, [summaryName summaryExt]); - sessionReportLocation = fullfile(sessionFolder, [sessionName sessionExt]); - tempReportLocation = fullfile(sessionFolder, [wrapperScriptName '.pdf']); - relativeReportLocation = getRelativePath(summaryFolder, sessionFolder, sessionName, sessionExt); - - fprintf('Summary: %s session: %s\n', summaryFolder, sessionFolder); - fprintf('Relative report location %s \n', relativeReportLocation); - - if isempty(EEG) || ~isfield(EEG, 'etc') || ~isfield(EEG.etc, 'noiseDetection') - error('publishPrepReport:PrepNotRun', ... - ['EEG.etc must contain PREP informational structures to ' ... - 'run reports --- run PREP first']); - end - - if publishOn - % Save variables to temp file - dataPath = fullfile(tempdir, 'prepReportData.mat'); - save(dataPath, 'EEG', 'consoleFID', 'relativeReportLocation', 'summaryReportLocation'); - - % Write wrapper script - - wrapperPath = fullfile(tempdir, [wrapperScriptName '.m']); - fid = fopen(wrapperPath, 'w'); - fprintf(fid, 'load(''%s'');\n', dataPath); - fprintf(fid, 'summaryFile = fopen(summaryReportLocation, ''a+'', ''n'', ''UTF-8'');\n'); - fprintf(fid, 'prepReport(EEG, summaryFile, consoleFID, relativeReportLocation);\n'); - fprintf(fid, 'fclose(summaryFile);\n'); - fprintf(fid, 'clear consoleFID relativeReportLocation summaryReportLocation summaryFile tmpEEG;\n'); - fclose(fid); - - % Add tempdir to MATLAB path so publish can find the script - addpath(tempdir); - - % Set publish options - publish_options.outputDir = sessionFolder; - publish_options.maxWidth = 800; - publish_options.format = 'pdf'; - publish_options.showCode = false; - - % Publish report - publish(wrapperScriptName, publish_options); - - rmpath(tempdir); - movefile(tempReportLocation, sessionReportLocation); - close all; - delete(wrapperPath); - delete(dataPath); - else - summaryFile = fopen(summaryReportLocation, 'a+', 'n', 'UTF-8'); - if summaryFile == -1 - error('publishPrepReport:BadSummaryFile', ... - 'Failed to open summary file %s', summaryReportLocation); - end - prepReport(EEG, summaryFile, consoleFID, relativeReportLocation); - fclose(summaryFile); - clear consoleFID relativeReportLocation summaryReportLocation summaryFile tmpEEG; - end - -end - -function relativePath = getRelativePath(summaryFolder, sessionFolder, sessionName, sessionExt) - relativePath = relativize(getCanonicalPath(summaryFolder), ... - getCanonicalPath(sessionFolder)); - relativePath = getCanonicalPath(relativePath); - while true - relativePathNew = strrep(relativePath, '\\', '\'); - if length(relativePathNew) == length(relativePath) - break; - end - relativePath = relativePathNew; - end - relativePath = strrep(relativePath, '\', '/'); - relativePath = [relativePath sessionName sessionExt]; -end - -function canonicalPath = getCanonicalPath(canonicalPath) - if canonicalPath(end) ~= filesep - canonicalPath = [canonicalPath, filesep]; - end -end - -function cleanup(backupFile, currentFile, warningsState) - restoreEEGOptions(backupFile, currentFile); - warning(warningsState); -end +function [] = publishPrepReport(EEG, summaryFilePath, sessionFilePath, ... + consoleFID, publishOn) +% Create a published report from the PREP pipeline. +% +% Note: In addition to creating a report for the EEG, it appends a +% summary of the file to an existing summary file. This enables the +% function to be called successfully on a collection and creates a summary +% of the collection. +% +% Parameters: +% EEG EEGLAB structure with the EEG.etc.noiseDetection +% structure created by the PREP pipeline +% summaryFilePath File name including path of the summary file +% sessionFilePath File name including path of the individual report +% consoleID Open file descriptor for echoing output (usually 1 +% indication the Command Window). +% publishOn If true (default), report is published and +% figures are closed. If false, output and figures +% are displayed in the normal way. The figures +% are not closed. This option is useful when +% you want to manipulate the figures in some way. +% +% Output: +% If the publish option is on, this function will create a report +% for the EEG and will append a summary to a specified summary file. +% If the publish option is off, the function will just run the +% prepReport script. +% +% Author: Kay Robbins, UTSA, March 2015. +% +% +%% Handle the parameters + if (nargin < 4) + error('publishPrepReport:NotEnoughParameters', ... + ['Usage: publishPrepReport(EEG, summaryFilePath, ' ... + 'sessionFilePath, consoleFID, publishOn)']); + elseif nargin < 5 || isempty(publishOn) + publishOn = true; + end + +%% Make sure that EEGLAB is working in double precision + [backupOptionsFile, currentOptionsFile, warningsState] = setupForEEGLAB(); + finishup = onCleanup(@() cleanup(backupOptionsFile, currentOptionsFile, ... + warningsState)); + +%% Setup up files and assign variables needed for publish in base workspace + wrapperScriptName = 'prepReportWrapper'; + [summaryFolder, summaryName, summaryExt] = fileparts(summaryFilePath); + [sessionFolder, sessionName, sessionExt] = fileparts(sessionFilePath); + summaryReportLocation = fullfile(summaryFolder, [summaryName summaryExt]); + sessionReportLocation = fullfile(sessionFolder, [sessionName sessionExt]); + tempReportLocation = fullfile(sessionFolder, [wrapperScriptName '.pdf']); + relativeReportLocation = getRelativePath(summaryFolder, sessionFolder, sessionName, sessionExt); + + fprintf('Summary: %s session: %s\n', summaryFolder, sessionFolder); + fprintf('Relative report location %s \n', relativeReportLocation); + + if isempty(EEG) || ~isfield(EEG, 'etc') || ~isfield(EEG.etc, 'noiseDetection') + error('publishPrepReport:PrepNotRun', ... + ['EEG.etc must contain PREP informational structures to ' ... + 'run reports --- run PREP first']); + end + + if publishOn + % Save variables to temp file + dataPath = fullfile(tempdir, 'prepReportData.mat'); + save(dataPath, 'EEG', 'consoleFID', 'relativeReportLocation', 'summaryReportLocation'); + + % Write wrapper script + + wrapperPath = fullfile(tempdir, [wrapperScriptName '.m']); + fid = fopen(wrapperPath, 'w'); + fprintf(fid, 'load(''%s'');\n', dataPath); + fprintf(fid, 'summaryFile = fopen(summaryReportLocation, ''a+'', ''n'', ''UTF-8'');\n'); + fprintf(fid, 'prepReport(EEG, summaryFile, consoleFID, relativeReportLocation);\n'); + fprintf(fid, 'fclose(summaryFile);\n'); + fprintf(fid, 'clear consoleFID relativeReportLocation summaryReportLocation summaryFile tmpEEG;\n'); + fclose(fid); + + % Add tempdir to MATLAB path so publish can find the script + addpath(tempdir); + + % Set publish options + publish_options.outputDir = sessionFolder; + publish_options.maxWidth = 800; + publish_options.format = 'pdf'; + publish_options.showCode = false; + + % Publish report + publish(wrapperScriptName, publish_options); + + rmpath(tempdir); + movefile(tempReportLocation, sessionReportLocation); + close all; + delete(wrapperPath); + delete(dataPath); + else + summaryFile = fopen(summaryReportLocation, 'a+', 'n', 'UTF-8'); + if summaryFile == -1 + error('publishPrepReport:BadSummaryFile', ... + 'Failed to open summary file %s', summaryReportLocation); + end + prepReport(EEG, summaryFile, consoleFID, relativeReportLocation); + fclose(summaryFile); + clear consoleFID relativeReportLocation summaryReportLocation summaryFile tmpEEG; + end + +end + +function relativePath = getRelativePath(summaryFolder, sessionFolder, sessionName, sessionExt) + relativePath = relativize(getCanonicalPath(summaryFolder), ... + getCanonicalPath(sessionFolder)); + relativePath = getCanonicalPath(relativePath); + while true + relativePathNew = strrep(relativePath, '\\', '\'); + if length(relativePathNew) == length(relativePath) + break; + end + relativePath = relativePathNew; + end + relativePath = strrep(relativePath, '\', '/'); + relativePath = [relativePath sessionName sessionExt]; +end + +function canonicalPath = getCanonicalPath(canonicalPath) + if canonicalPath(end) ~= filesep + canonicalPath = [canonicalPath, filesep]; + end +end + +function cleanup(backupFile, currentFile, warningsState) + restoreEEGOptions(backupFile, currentFile); + warning(warningsState); +end diff --git a/PrepPipeline/reporting/showPipelineDefaults.m b/PrepPipeline/reporting/showPrepDefaults.m similarity index 100% rename from PrepPipeline/reporting/showPipelineDefaults.m rename to PrepPipeline/reporting/showPrepDefaults.m diff --git a/PrepPipeline/utilities/blasst/README.md b/PrepPipeline/utilities/blasst/README.md deleted file mode 100644 index 5430be1..0000000 --- a/PrepPipeline/utilities/blasst/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# blasst -BLASST: Band Limited Atomic Sampling with Spectral Tuning Toolbox - -This option is still under testing. diff --git a/PrepPipeline/utilities/blasst/blasst.m b/PrepPipeline/utilities/blasst/blasst.m deleted file mode 100644 index 3a00a1a..0000000 --- a/PrepPipeline/utilities/blasst/blasst.m +++ /dev/null @@ -1,227 +0,0 @@ -function [x,varargout] = blasst(x,lineFrequencies,frequencyRanges,samplingRate,varargin) -% blasst(): EEGLAB helper function for OCW line noise removal. -% Takes as input an array of signals x, along with relevant parameters, and -% performs BLASST filtering at specified frequencies. For each specified -% frequency, blasst() iteratively calls blasst_internal() and then uses -% blasst_test() to test for convergence based on the distributions of -% convolution coefficients in the target and surrounding frequency bands. -% -% INPUT: -% x an [n,N] array of n signals of length N. -% lineFrequencies an array of target frequencies (not normalized). -% frequenyRanges an array of target frequency ranges. Must be the same -% size as lineFrequencies. -% samplingRate the sampling rate of the signal. -% varargin optional 'key',value pairs: -% 'key' -% [default] purpose -% 'Scale' -% [2^(log2(samplingRate)+2)] Manually set the scale that -% indicates the spread of Gabor atoms. -% 'ContinuousEpochs' -% [0] If x is epoched, but epochs are temporally adjacent, -% setting to 1 will flatten x for processing. Otherwise, -% blasst is run on individual epochs. -% 'Verbose' -% [1] When on, progress is printed on command line. -% 'Resolution' -% [2] May be an integer value >= 1, sets 'resolution' in blasst. -% Specificies density of Gabor atoms. May also be an array of -% integer values of size(lineFrequencies). -% 'MaxIterations' -% [50] Maximum number of external iterations of blasst run on -% each frequency. May be either a scalar integer or array of -% integers of size(lineFrequencies). -% 'ManualOffset' -% [log2(scaleBases)+1] A scalar value that offsets the -% arrangement of Gabor atoms at each iteration of blasst. -% 'Channels' -% [1:size(x,1)] An array of integers indexing channels to -% be computed. Allows manually selection of channels for -% processing. -% -% OUTPUT: -% x the processed, or ``cleaned'' signal. -% varargout{1} the aggregate of the target signal feature removed, an -% array of size(x). -% -% DEPENDENCIES: -% blasst_internal() primary line noise removal algorithm. -% blasst_test() convergence test for iterative blasst algorithm. -% -% EXAMPLE: -% Suppose we wish to remove line noise frequency at 60 and 120 Hz, and the -% noise is mostly stationary at 120 Hz but non-stationary varying by about -% 2 Hz, around 60 Hz. Then we might call the method as: -% >> x = blasst(x,[60,120],[2,.25],); -% -% If we want to use more densely packed Gabor atoms, we could call: -% >> x = blasst(x,[60,120],[2,.25],,'Resolution',4); -% -% AUTHOR: Kenneth Ball, 2015. -% -% IF YOU FIND BLASST USEFUL IN YOUR WORK, PLEASE CITE: -% -% Ball, K. R., Hairston, W. D., Franaszczuk, P. J., Robbins, K. A., -% BLASST: Band Limited Atomic Sampling with Spectral Tuning with -% Applications to Utility Line Noise Filtering, [Under Review]. -% -% Copyright 2015 Kenneth Ball -% -% Licensed under the Apache License, Version 2.0 (the "License"); -% you may not use this file except in compliance with the License. -% You may obtain a copy of the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, -% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% See the License for the specific language governing permissions and -% limitations under the License. - -% Set defaults: -flattenData = 0; -verbose = 1; -resolution = 2; -maxIterations = 50; % Default is high so that the method generally will generally converge. -channels = 1:size(x,1); - -% Adjust for optional inputs -if (nargin> 1) - if (nargin> 4 && rem(nargin,2) == 1) - if length(varargin) == 1 - varargin = varargin{1}; - else - fprintf('blasst(): Optional key and value pairs do not match.') - return - end - end - - for ii = 1:2:length(varargin) - key = varargin{ii}; - val = varargin{ii+1}; - switch key - case 'SamplingRate' - samplingRate = val; - case 'Scale' - scale = val; - case 'ContinuousEpochs' - flattenData = val; - case 'Verbose' - verbose = val; - case 'Resolution' - resolution = val; - case 'MaxIterations' - maxIterations = val; - case 'ManualOffset' - manualOffset = val; - case 'Channels' - channels = val; - end - end -end - -if length(frequencyRanges) == 1 - frequencyRanges = ones(size(lineFrequencies))*frequencyRanges; -elseif length(frequencyRanges) ~= length(lineFrequencies) - error('Number of specified noise frequencies does not match number of specified range values.'); -end - -if length(resolution) == 1 - resolution = ones(size(lineFrequencies))*resolution; -elseif length(resolution) ~= length(lineFrequencies) - error('Number of specified noise resolutions does not match number of specified range values.'); -end - -if length(maxIterations) == 1 - maxIterations = ones(size(lineFrequencies))*maxIterations; -elseif length(maxIterations) ~= length(lineFrequencies) - error('Number of specified max iterations does not match number of specified range values.'); -end - -if ~exist('samplingRate','var') - error('No sampling rate specified.'); -elseif isempty(samplingRate) || samplingRate == 0 - error('Null or zero sampling rate specified.'); -end - -if ~exist('scaleBases','var') - scale = 2^(log2(samplingRate)+2); -end - -if ~exist('manualOffset','var') - manualOffset = log2(scale)+1; -end - -if flattenData - dataSizeTemp = size(x); - x = reshape(x,size(x,1),size(x,2)*size(x,3)); -end - -% Initialize holders for fitted noise and cleaned signals. -y = zeros(size(x)); -yTemp = zeros(size(x)); -xTemp = zeros(size(x)); - -countTrack = 1; - -for ii = 1:length(lineFrequencies) - - if verbose - fprintf(1,'Frequency: %d Hz\n',lineFrequencies(ii)); - end - for jj = channels - if verbose - fprintf(1,'Computing Channel: '); - end - % BDist = Inf; - [BDist,~,~] = blasst_test(reshape(x(jj,:,:),1,size(x,2)*size(x,3)),lineFrequencies(ii),frequencyRanges(ii),samplingRate,scale); - for mm = 1:maxIterations(ii) - if verbose - fprintf(1,'\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b% 3i Pass: % 3i',jj,mm); - end - % Run blasst_internal for each epoch of data in the EEG struct. - for kk = 1:size(x,3) - [yTemp(jj,:,kk),xTemp(jj,:,kk)] = blasst_internal(x(jj,:,kk),lineFrequencies(ii),frequencyRanges(ii),samplingRate,scale,resolution(ii),(mm-1)*manualOffset); - % y(jj,:,kk) = y(jj,:,kk) + tempY; - % EEG.data(jj,:,kk) = tempX; - end - - % Compute Bhatt. distance for testing convergence. Overwrite - % BDist1, then compare to BDist (from the last pass). If BDist1 - % exceeds BDist, we presume we have passed the minimum distance - % between the distributions, and we should halt the algorithm - % for this channel and frequency, retaining only previous - % passes. - - [BDist1,maxFlag,~] = blasst_test(reshape(xTemp(jj,:,:),1,size(xTemp,2)*size(xTemp,3)),lineFrequencies(ii),frequencyRanges(ii),samplingRate,scale); - countTrack = countTrack+1; - if BDist1 >= BDist && ~maxFlag % && mm > 1 - if verbose - fprintf(1,'\nBreak at pass % 2i\n',mm-1); - end - break - else - BDist = BDist1; - y(jj,:,:) = y(jj,:,:) + yTemp(jj,:,:); - x(jj,:,:) = xTemp(jj,:,:); - end - end - if verbose - fprintf(1,'\n'); - end - end - if verbose - fprintf(1,'\n'); - end - -end - -varargout{1} = y; % This is the aggregate of all line noise that was removed. That is, y+EEG.data is the original dataset. - -if flattenData - x = reshape(x,dataSizeTemp(1),dataSizeTemp(2),dataSizeTemp(3)); -end - -end diff --git a/PrepPipeline/utilities/blasst/blasst_documentation.tex b/PrepPipeline/utilities/blasst/blasst_documentation.tex deleted file mode 100644 index 13a933e..0000000 --- a/PrepPipeline/utilities/blasst/blasst_documentation.tex +++ /dev/null @@ -1,163 +0,0 @@ -\documentclass[11pt]{article} - -\usepackage{amsmath} -\usepackage{amsfonts} -\usepackage{amsthm} -\usepackage{fullpage} -\usepackage[square,numbers]{natbib} -\usepackage{hyperref} - -\usepackage[percent]{overpic} -\usepackage{graphicx} -\usepackage{cmap} - - -%%Theorems -\newtheorem{theorem}{Theorem}[section] -\newtheorem{corollary}[theorem]{Corollary} -\newtheorem{proposition}[theorem]{Proposition} -\newtheorem{claim}[theorem]{Claim} -\newtheorem{lemma}[theorem]{Lemma} -\newtheorem{definition}[theorem]{Definition} -\newtheorem{axiom}[theorem]{Axiom} -\newtheorem{problem}[theorem]{Problem} - -\theoremstyle{remark} -\newtheorem*{remark}{Remark} - -%%% Comments and Todos -\newcommand{\ppar}[1]{\noindent{\em{#1}}} -\newcommand{\comment}[1]{\par\noindent{\raggedright\texttt{#1} -\par\marginpar{\textsc{Comment}}}} -\newcommand{\todo}[1]{\vspace{5 mm}\par \noindent -\marginpar{\textsc{ToDo}} -\framebox{\begin{minipage}[c]{0.98\columnwidth} -\tt #1 \end{minipage}}\vspace{5 mm}\par} - -%\usepackage{parskip} - -\title{BLASST: A MATLAB toolbox for filtering long-time, nonstationary signal artifacts.} -\author{Kenneth Ball} -\date{September, 2015} - -\begin{document} - -\maketitle - -\section*{Preliminaries} -If you find this tool/algorithm useful, please cite our associated paper: \\ \ \\ - -\noindent Ball, K.~R., Hairston, W.~D., Franaszczuk, P.~J., Robbins, K.~A., BLASST: Band Limited Atomic Sampling with Spectral Tuning with Applications to Utility Line Noise Filtering, [Under Review]. \\ \ \\ - -\section{Overview of BLASST} -Band Limited Atomic Sampling with Spectral Tuning (BLASST) is an algorithm developed to filter relatively long-time, non-stationary signal features, especially the 50/60 Hz utility line noise that is almost ubiquitious in sensitive experiments. Our motivating use case is ambulatory EEG experiments where gamma-range neural features may be of interest. We observe that most experimenters either notch filter about 50/60 Hz, or low-pass filter signals at some threshold below 50/60 Hz in order to remove the often dominant line noise features from their collected signals. Especially in cases where the utility line noise is highly non-stationary, due to either fluctuations in power generation atht the utility level or non-linear effects caused by interfering electronics in the laboratory, a relatively wide notch-filter may be required to completely remove the noise, which in turn causies a complete loss of otherwise useful information in the targeted spectral band. - -BLASST attempts to fit non-stationary line-noise more flexibly by attempting to reconstruct the dominant signal in the supplied target frequency range with a set of Gabor atoms arranged in time so that their envoloping Gaussian functions (approximately) add up to a partition of unity. BLASST iterates this fitting approach until a stopping criterion is reached. Information from spectral bands outside of the user-supplied target bands is leveraged to (1) modulate the fit at each iteration and (2) determine a stopping criterion based on the distribution of the amplitudes of complex Gabor atoms drawn from inside and outside the target frequency spectrum. - -\section{blasst():} -\verb![signalOut , varargout] = blasst(signalIn , lineFrequencies , frequencyRanges , ...! \verb!samplingRate , varargin);! - -\subsection{Input:} -\begin{itemize} -\item \verb!signalIn! :: an $n\times N$ array of signal data. The signals are of length $N$, and there are $n$ total channels of data (so generally $n << N$). - -\item \verb!lineFrequencies! :: an array of target frequencies (not normalized). - -\item \verb!frequencyRanges! :: an array of target frequency ranges. Must be the same size as \verb!lineFrequencies!. - -\item \verb!samplingRate! :: the sampling rate of the signal. - -\item \verb!varargin! :: optional 'key',value pairs: - -\begin{itemize} - -\item \verb~'key'~ - [default] purpose -\item \verb~'Scale'~ - [\verb~2^(log2(samplingRate)+2)~] Manually set the scale that - indicates the spread of Gabor atoms. -\item \verb~'ContinuousEpochs'~ - [\verb~0~] If x is epoched, but epochs are temporally adjacent, - setting to 1 will flatten x for processing. Otherwise, - blasst is run on individual epochs. -\item \verb~'Verbose'~ - [\verb~1~] When on, progress is printed on command line. -\item \verb~'Resolution'~ - [\verb~2~] May be an integer value $>= 1$, sets 'resolution' in blasst. - Specificies density of Gabor atoms. May also be an array of - integer values of \verb~size(lineFrequencies)~. -\item \verb~'MaxIterations'~ - [\verb~50~] Maximum number of external iterations of blasst run on - each frequency. May be either a scalar integer or array of - integers of \verb~size(lineFrequencies)~. -\item \verb~'ManualOffset'~ - [\verb~log2(scaleBases)+1~] A scalar value that offsets the - arrangement of Gabor atoms at each iteration of blasst. -\item \verb~'Channels'~ - [\verb~1:size(x,1)~] An array of integers indexing channels to - be computed. Allows manually selection of channels for - processing. - -\end{itemize} - -\end{itemize} - -\subsection{Output} -\begin{itemize} - -\item \verb!signalOut! :: the processed, or ``cleaned'' signal, an array of \verb~size(signalIn)~. -\item \verb!varargout{1}! :: the aggregate of the target signal feature removed, an array of \verb!size(signalIn)!. - -\end{itemize} - -\subsection{Basic Use Cases} - -\begin{itemize} -\item Suppose you have collected signals sampled at 512 Hz, and you wish to identify and remove 60 Hz line noise. The line noise seems to be relatively stationary in frequency, so you only target features between 59.75 and 60.25 Hz. Then you may run: - -\verb~ >>signalOut = blasst(signalIn,60,0.25,512);~ - -If you would like to return the noise removed \verb~noise~, you may run: - -\verb~ >>[signalOut,noise] = blasst(signalIn,60,0.25,512);~ - -\item If you would like to remove harmonics of 60 Hz up to 256 Hz (the Nyquist frequency), you may run: - -\verb~ >>signalOut = blasst(signalIn,[60,120,180,240],0.25,512);~ - -\item Suppose you observe high non-stationarity in frequency at 120 Hz, but tight bounds on spectral power at the other harmonics of 60 Hz. Increasing the target frequency range increases the time required to search for best fit Gabor atoms, and also increases the possiblity of overfitting. Thus, it is preferable to only increase the target frequency range value for target frequencies where it is requried. In this case, you may run: - -\verb~ >>signalOut = blasst(signalIn,[60,120,180,240],[0.25,2,0.25,0.25],512);~ - -In this case, blasst() will fit features between $60\pm 0.25$, $120 \pm 2$, $180\pm 0.25$, and $240\pm 0.25$ Hz. - -\end{itemize} - -\subsection{Advanced Use Cases} - -\begin{itemize} - -\item The resolution of the fit at each iteration can be increased (or decreased) from the default value of $k = 2$ by using a key value pair \verb~ 'Resolution',k~ where \verb~k~ is an integer value greather than or equal to 1. In theory, higher resoltuion arrangements of Gabor atoms (more tightly packed) should allow for my fine tuned temporal flexibility. Of course, the Gabor atoms themselves should already be very spread out, so this may not do much. To increase the resolution to $k = 4$, you may run: - -\verb~ >>signalOut = blasst(signalIn,60,0.25,512,'Resolution',4);~ - -\item The scales of Gabor atoms can be manually adjusted. Higher scales allow for more specificity in frequencies, but less resolution in time; lower scales allow for more specificity in time, but less resolution in frequency. Scales are input as the number of time samples; the default value corresponds to 4 seconds (regardless of sampling rate). Suppose you wanted to use Gabor atoms with scales of 3 seconds, and the sampling rate was 250 Hz. Then you could run: - -\verb~ >>signalOut = blasst(signalIn,60,0.25,256,'Scale',3*256);~ - -\end{itemize} - -\subsection{Alterantive Use Cases} - -\begin{itemize} - -\item We believe BLASST may be useful for other types of filtering were flexibility and robustness to short-time signal features is desirable. For example suppose you were interested in the appearance of alpha spindles in a 512 Hz EEG signal between 9 and 14 Hz. Further, suppose you expect such features to appear in bursts of at least 0.5 seconds. Then you could run: - -\verb~ >>[signalOut,alpha] = blasst(signalIn,11.5,2.5,512,'Scale',512/2,'Resolution',4);~ - -Then \verb~alpha~ will be returned as the alpha features. Theoretically, BLASST should return the desired alpha features while reducing artifacts in the target spectrum caused by spectral bleed of more temporally localized spikes of activity, such as muscle movements. - - -\end{itemize} - -\end{document} diff --git a/PrepPipeline/utilities/blasst/blasst_internal.m b/PrepPipeline/utilities/blasst/blasst_internal.m deleted file mode 100644 index a3527c0..0000000 --- a/PrepPipeline/utilities/blasst/blasst_internal.m +++ /dev/null @@ -1,227 +0,0 @@ -function [y,x] = blasst_internal(x,f,r,sR,scale,rez,manualOffset) -% blasst_internal(): An iteration of BLASST feature fitting. -% Takes a 1-dimensional signal and fits a series of Gabor atoms (arranged -% so that the enveloping Gaussian functions add up to a partition of unity) -% to try and remove the target frequency (f) within a range (\pm r), while -% respecting the time-frequency power distribution of the surrounding -% spectral bands. -% -% INPUT: -% x is [1,N] time series signal. -% f is the target frequency (a scalar, NOT normalized) ex. 60 for 60 Hz -% r is the target frequency range within which we seek to remove noise. -% For example, if line noise is not stationary, we might seek to -% remove noise between 57 and 63 Hz, in which case r = 3. -% sR is the sampling rate of the signal. -% scale is the integer scale for Gabor atoms. -% rez is an integer >= 1 that specifies the density of Gabor atoms in the -% partition of unity. rez must take an integer value so that Gaussian -% functions add up to unity. -% manualOffset is a scalar that offsets the centers of atoms at the outset -% of computation. -% -% OUTPUT: -% y the [1,N] time course of signal features removed from x. -% x the [1,N] transformed signal. -% -% AUTHOR: Kenneth Ball, 2015. -% -% IF YOU FIND BLASST USEFUL IN YOUR WORK, PLEASE CITE: -% -% Ball, K. R., Hairston, W. D., Franaszczuk, P. J., Robbins, K. A., -% BLASST: Band Limited Atomic Sampling with Spectral Tuning with -% Applications to Utility Line Noise Filtering, [Under Review]. -% -% Copyright 2015 Kenneth Ball -% -% Licensed under the Apache License, Version 2.0 (the "License"); -% you may not use this file except in compliance with the License. -% You may obtain a copy of the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, -% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% See the License for the specific language governing permissions and -% limitations under the License. - - - -N = length(x); -y = zeros(size(x)); -n2 = scale*2; % half-window length -n = 2*n2+1; % win length -win = (-n2):(n2); - -% timejump and estimate amplitude scaling for partition of unity of Gaussian functions: -timeJump = scale/rez/sqrt(pi); -R = -(rez*10):(rez*10); -scalingFactor = 1./sum(exp(-R.^2/rez.^2)); - -% spec frequency ranges: -fs = sR*(0:(1/(2*scale)):.5); -fs = fs(intersect(find(fs>=(f-r)),find(fs<=(f+r)))); - -wavelets = complexGabor(fs/sR,scale,0,win); % wavelets is [length(fs),length(win)] = [length(fs),n]; - -% spec test frequency ranges: -tfs = sR*(0:(1/(2*scale)):.5); -tfsl = tfs(intersect(find(tfs>=(f-3*r)),find(tfs<=(f-2*r)))); -tfsr = tfs(intersect(find(tfs>=(f+2*r)),find(tfs<=(f+3*r)))); -tfs = [tfsl,tfsr]; -testWavelets = complexGabor(tfs/sR,scale,0,win); - -modulator = buildPowerModulator([zeros(1,2*n),x,zeros(1,2*n)],testWavelets,fs,tfs); - -% Build the windowed signal array: -centers = (1+manualOffset):timeJump:N; % Interior centers. -leftCenters = centers(1); -centers = centers(2:end); -while round(leftCenters(1)) > -n2+1 - leftCenters = cat(2,leftCenters(1)-timeJump,leftCenters); -end -while round(leftCenters(end)) < n2 - leftCenters = cat(2,leftCenters,leftCenters(end)+timeJump); - centers = centers(2:end); -end -rightCenters = centers(end); -centers = centers(1:end-1); -while round(rightCenters(end)) < N+n2 - rightCenters = cat(2,rightCenters,rightCenters(end)+timeJump); -end -while round(rightCenters(1)) > N-n2 - rightCenters = cat(2,rightCenters(1)-timeJump,rightCenters); - centers = centers(1:end-1); -end -centers = round([leftCenters,centers,rightCenters]); -weights = scale/2*(erf(sqrt(pi)*(N-centers)/scale)-erf(sqrt(pi)*(1-centers)/scale))/scale; -weights = weights( 2:(end-1) ); -centers = centers( 2:(end-1) ); - -% Pad x and y, initialize X (holder of signal sections on temporal support of atoms), and initizlize the power modulator: -x = [zeros(1,2*n),x,zeros(1,2*n)]; -y = [zeros(1,2*n),y,zeros(1,2*n)]; -X = zeros(length(centers),n); -pM = zeros(size(X,1),size(modulator,1)); % the power modulator - -% Adjust the "centers" index to account for the padding by n -centers = centers + 2*n; - -% Build X and the power modulator. -for jj = 1:length(centers) - X(jj,:) = x((-n2:n2)+centers(jj)); - pM(jj,:) = mean(modulator(:,round( ((-scale/(2*rez)):(scale/(2*rez)))+centers(jj))),2)'; -end - -% Calculate phase and amplitude of fitted gabor atoms: -phase = -angle( X*wavelets.'); %[windowCount,length(fs)] -realWavelets = zeros(size(X,1),n,length(fs)); -amplitude = zeros(size(phase)); -for jj = 1:length(fs) - % Compute the real wavelets. - realWavelets(:,:,jj) = real(exp(1i*phase(:,jj))*wavelets(jj,:)); %[windowCount,1]*[1,n] = [windowCounts,n] - % Modify the real wavelet amplitude by the power modulator. - amplitude(:,jj) = sum(X.*squeeze(realWavelets(:,:,jj)),2)-pM(:,jj) ; - % Set negative modified amplitudes to zero. - amplitude(:,jj) = amplitude(:,jj).*(amplitude(:,jj) > 0); -end -% Find the best fit atom after power modulation. -[amps,whichFreq] = max(amplitude,[],2); % amps and whichFreq are [windowCount,1] - - -% Put the best fit atom into the the 1st instance of the 3rd index of -% realWavelets. -for jj = 1:size(X,1) - realWavelets(jj,:,1) = realWavelets(jj,:,whichFreq(jj)); -end -% Retain only the best fit atoms. -realWavelets = squeeze(realWavelets(:,:,1)); - -% Rescale atoms according to partition of unity. -realWavelets = bsxfun(@times,scalingFactor*amps./weights',realWavelets)*sqrt(2); - -% Update x and y with computed realWavelets atoms: -for jj = 1:length(centers) - x((-n2:n2)+centers(jj)) = x((-n2:n2)+centers(jj)) - realWavelets(jj,:); - y((-n2:n2)+centers(jj)) = y((-n2:n2)+centers(jj)) + realWavelets(jj,:); -end -x = x((2*n+1):(end-2*n)); -y = y((2*n+1):(end-2*n)); - -end - -function modulator = buildPowerModulator(x,testDict,targetFreqs,testFreqs) - -C = multiConvolveFFT(testDict,x); - -CmodWeights = zeros(length(targetFreqs),length(testFreqs)); - -for kk = 1:length(targetFreqs) - CmodWeights(kk,:) = targetFreqs(kk)-testFreqs; -end -CmodWeights = abs(1./CmodWeights); -CmodWeights = bsxfun(@times,CmodWeights,1./sum(CmodWeights,2)); - -modulator = sqrt(exp( CmodWeights*log(abs(C).^2) )); - - -end - -function gaborFun = complexGabor(f,s,u,time) -% can output multiple gaborFuns: gaborFun is [p,n] -% time is [1,n] -% s,f are [1,p] -s = s'; % [p,1] -f = f'; % [p,1] -if ~isinf(s) - g1 = bsxfun(@times,2^(1/4)/sqrt(s),exp(-pi/s^2*(time-u).^2)); -% goo = length(time)-(s+1); -% g1 = [zeros(1,goo/2),dpss(s+1,3,1)',zeros(1,goo/2)]; - gaborFun = bsxfun(@times,g1,exp(1i*2*pi*f*(time-u))); -else - gaborFun = exp(1i*2*pi*f*(time-u)); -end -% gaborFun = bsxfun(@times,gaborFun,(1./sqrt(sum(abs(gaborFun.').^2)))'); -% if f == 0 -% gaborFun = gaborFun./sqrt(2); -% end - -end - -function [C,varargout] = multiConvolveFFT(filters,y) -% filters are an [k,n] bank of functions, where n is length of each filter -% and k is the number of filters. y is a [1,N] signal vector. -% returns [k,N] (absolute value) convolutions. -% optionaly returns the phase, or argument, of the complex convolutions. - -% pad = size(filters,2); -% nn = 3*pad+length(y)-1; -% yF = fft([zeros(1,pad),y,zeros(1,pad)],nn); -% fF = fft(filters,nn,2); -% C = ifft(bsxfun(@times,yF,fF),[],2); - -% % Convolve left and right edges: -% n2 = ceil((size(filters,2)-1)/2); -% n = size(filters,2); -% yL = [-fliplr(y(1:n)),y(1:n)]; -% yR = [y((end-n+1):end),-fliplr(y((end-n+1):end))]; -% for ii = 1:n2 -% conL(:,ii) = sum(bsxfun(@times,filters,yL( (ii+n2):(ii+n2-1+n) )),2); -% conR(:,ii) = sum(bsxfun(@times,filters,yR( (ii):(ii-1+n) )),2); -% end - - - nn = size(filters,2) + length(y) ; % nn = n+N - yF = fft(y,nn); - fF = fft(filters,nn,2); - C = ifft(bsxfun(@times,yF,fF),[],2); - C = C(:,( floor(size(filters,2)/2+1) + (0:(length(y)-1)) )); - -% C(:,1:n2) = conL; -% C(:,(end-n2+1):end) = conR; - - varargout{1} = angle(C); - %C = abs(C); - -end diff --git a/PrepPipeline/utilities/blasst/blasst_test.m b/PrepPipeline/utilities/blasst/blasst_test.m deleted file mode 100644 index daf2d90..0000000 --- a/PrepPipeline/utilities/blasst/blasst_test.m +++ /dev/null @@ -1,135 +0,0 @@ -function [BDist,maxFlag,varargout] = blasst_test(x,f,r,sR,scale) -% blasst_test(): Tests for convergence of the BLASST line noise removal -% approach by comparing the Bhattahcharya distance between the target and -% test frequency bands. -% -% INPUT: -% x is [1,N] time series signal. -% f is the target frequency (a scalar, NOT normalized) ex. 60 for 60 Hz -% r is the target frequency range within which we seek to remove noise. -% For example, if line noise is not stationary, we might seek to -% remove noise between 57 and 63 Hz, in which case r = 3. -% sR is the sampling rate of the signal. -% scale is the scale of Gabor atoms to be compared. -% -% OUTPUT: -% BDist is the Bhattacharya distance between the weighted average -% probability distributions of the test and target bands. -% maxFlag is a flag that specifies whether or not the target -% distribution is trivially way to the right of the test. -% Helps to avoid numerical errors in convergence at early -% iterations of OCW_LNR. -% varargout{1} is a struct with fields: -% 'CHists' is the the probability distributions of the target -% spectral band. -% 'DHists' is the the probability distributions of the test spectral -% band. -% -% AUTHOR: Kenneth Ball, 2015. -% -% IF YOU FIND BLASST USEFUL IN YOUR WORK, PLEASE CITE: -% -% Ball, K. R., Hairston, W. D., Franaszczuk, P. J., Robbins, K. A., -% BLASST: Band Limited Atomic Sampling with Spectral Tuning with -% Applications to Utility Line Noise Filtering, [Under Review]. -% -% Copyright 2015 Kenneth Ball -% -% Licensed under the Apache License, Version 2.0 (the "License"); -% you may not use this file except in compliance with the License. -% You may obtain a copy of the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, -% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% See the License for the specific language governing permissions and -% limitations under the License. - -N = size(x,2); -n2 = scale*2; % half-window length -win = (-n2):(n2); - -% spec frequency ranges: -fs = sR*(0:(1/(2*scale)):.5); -fs = fs(intersect(find(fs>=(f-r)),find(fs<=(f+r)))); - -wavelets = complexGabor(fs/sR,scale,0,win); % wavelets is [length(fs),length(win)] = [length(fs),n]; - -% spec test frequency ranges: -tfs = sR*(0:(1/(2*scale)):.5); -tfsl = tfs(intersect(find(tfs>=(f-3*r)),find(tfs<=(f-2*r)))); -tfsr = tfs(intersect(find(tfs>=(f+2*r)),find(tfs<=(f+3*r)))); -tfs = [tfsl,tfsr]; -testWavelets = complexGabor(tfs/sR,scale,0,win); - -C = multiConvolveFFT(wavelets,x); -D = multiConvolveFFT(testWavelets,x); - -testWeights = zeros(length(fs),length(tfs)); %[targetFrequencies,testFrequencies] -for kk = 1:length(fs) - testWeights(kk,:) = fs(kk)-tfs; -end -testWeights = abs(1./testWeights); -testWeights = bsxfun(@times,testWeights,1./sum(testWeights,2)); - -D = abs(D); -C = abs(C); % Now C and D are real and same size: [ - -binCount = ceil(2*N^(1/3)); -[DHists,centers] = hist(log((abs(D).^2)'),binCount); %[binCount,lenegth(tfs)] vectors -DHists = DHists/N; - -DCompare = mean(DHists*testWeights',2); -CHists = hist(log((abs(C).^2)'),centers); -CHists = CHists/N; -CCompare = mean(CHists,2); - -[~,CMaxInd] = max(CCompare); -if CMaxInd == binCount % Presumably, CCompare distribution is somewhat skewed to the right and we are nowhere near convergence. - maxFlag = 1; -else - maxFlag = 0; -end - -BDist = -log(sum(sqrt(DCompare.*CCompare))+1e-8); -varargout{1}.CHist = CHists; -varargout{1}.DHist = DHists; -varargout{1}.CCompare = CCompare; -varargout{1}.DCompare = DCompare; -varargout{1}.centers = centers; - - -end - -function gaborFun = complexGabor(f,s,u,time) -% can output multiple gaborFuns: gaborFun is [p,n] -% time is [1,n] -% s,f are [1,p] -s = s'; % [p,1] -f = f'; % [p,1] -if ~isinf(s) - g1 = bsxfun(@times,2^(1/4)/sqrt(s),exp(-pi/s^2*(time-u).^2)); -% goo = length(time)-(s+1); -% g1 = [zeros(1,goo/2),dpss(s+1,3,1)',zeros(1,goo/2)]; - gaborFun = bsxfun(@times,g1,exp(1i*2*pi*f*(time-u))); -else - gaborFun = exp(1i*2*pi*f*(time-u)); -end - -end - -function [C,varargout] = multiConvolveFFT(filters,y) -% filters are an [k,n] bank of functions, where n is length of each filter -% and k is the number of filters. y is a [1,N] signal vector. -% returns [k,N] (absolute value) convolutions. -% optionaly returns the phase, or argument, of the complex convolutions. - nn = size(filters,2) + length(y); % nn = n+N - yF = fft(y,nn); - fF = fft(filters,nn,2); - C = ifft(bsxfun(@times,yF,fF),[],2); - C = C(:,( floor(size(filters,2)/2+1) + (0:(length(y)-1)) )); - varargout{1} = angle(C); - -end diff --git a/PrepPipeline/utilities/blasst/pop_blasst.m b/PrepPipeline/utilities/blasst/pop_blasst.m deleted file mode 100644 index f350c8c..0000000 --- a/PrepPipeline/utilities/blasst/pop_blasst.m +++ /dev/null @@ -1,86 +0,0 @@ -function [EEG,com] = pop_blasst(EEG,lineFrequencies,frequencyRanges,varargin) -% pop_blasst(): EEGLAB helper function for blasst filtering. -% Takes as input an EEGLAB EEG struct, along with relevant parameters, and -% calls blasst() for BLASST fitlering at specified frequencies. For each -% specified frequency, blasst() iteratively calls blasst_internal() and -% then uses blasst_test() to test for convergence based on the -% distributions of convolution coefficients in the target and surrounding -% frequency bands. -% -% AUTHOR: Kenneth Ball, 2015. -% -% IF YOU FIND BLASST USEFUL IN YOUR WORK, PLEASE CITE: -% -% Ball, K. R., Hairston, W. D., Franaszczuk, P. J., Robbins, K. A., -% BLASST: Band Limited Atomic Sampling with Spectral Tuning with -% Applications to Utility Line Noise Filtering, [Under Review]. -% -% Copyright 2015 Kenneth Ball -% -% Licensed under the Apache License, Version 2.0 (the "License"); -% you may not use this file except in compliance with the License. -% You may obtain a copy of the License at -% -% http://www.apache.org/licenses/LICENSE-2.0 -% -% Unless required by applicable law or agreed to in writing, software -% distributed under the License is distributed on an "AS IS" BASIS, -% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -% See the License for the specific language governing permissions and -% limitations under the License. - -com = ''; - -if nargin < 1 - help pop_blasst; - return -end -if ~isfield(EEG,'data') - error('Must specify signal(s) in EEG struct as: \n >>EEG.data = ;') -elseif isempty(EEG.data) - error('We can not BLASST nothing! EEG.data is empty.'); -end - -if isempty(lineFrequencies) - error('BLASST requires input target frequencies.'); -end - -if isempty(frequencyRanges) - error('BLASST requires input frequency ranges.'); -end - -if ~isfield(EEG,'srate') - error('Must specify sampling rate in input EEG struct as: \n >>EEG.srate = ;') -elseif isempty(EEG.srate) - error('BLASST must have a sampling rate! EEG.srate is empty.') -end - -EEG.data = blasst(EEG.data,lineFrequencies,frequencyRanges,EEG.srate,varargin); -foo = []; -% Process varargin cell into string of name value pairs. -if ~isempty(varargin) - if ~rem(length(varargin),2) - foo = ','; - if ischar(varargin{2}) - foo = [foo,'''',varargin{1}, ''', ''',varargin{2},'''']; - else - foo = [foo,'''',varargin{1}, ''', ',num2str(varargin{2})]; - end - for ii = 3:2:length(varargin); - if ischar(varargin{ii+1}) - foo = [foo, ', ''', varargin{ii}, ''', ''', varargin{ii+1},'''']; - else - foo = [foo, ', ''', varargin{ii}, ''', ', num2str(varargin{ii+1})]; - end - end -% foo = [foo,' }']; - else - error('Name value pairs do not match.') - end -end - -com = sprintf('%s = pop_blasst(%s,%s,%s%s);',inputname(1),inputname(1),mat2str(lineFrequencies),mat2str(frequencyRanges),foo); - - - -end diff --git a/PrepPipeline/utilities/blasstLineNoise.m b/PrepPipeline/utilities/blasstLineNoise.m deleted file mode 100644 index 5eda0fe..0000000 --- a/PrepPipeline/utilities/blasstLineNoise.m +++ /dev/null @@ -1,67 +0,0 @@ -function [signal, lineNoiseOut] = blasstLineNoise(signal, lineNoiseIn) -% Remove sharp spectral peaks from signal using Sleppian filters -% -% Usage: -% signal = cleanLineNoise(signal) -% [signal, lineNoiseOut] = hcleanLineNoise(signal, lineNoiseIn) -% -% Parameters: -% signal Structure with .data and .srate fields -% lineNoiseIn Input structure with fields described below -% -% Structure parameters (lineNoiseIn): -% fPassBand Frequency band used (default [0, Fs/2] = entire band) -% Fs Sampling frequency -% fScanBandWidth +/- bandwidth centered on each f0 to scan for significant -% lines (TM) -% lineFrequencies Line frequencies to be removed (default -% [60, 120, 180, 240, 300]) -% lineNoiseChannels Channels to remove line noise from (default -% size(data, 1)) -% maximumIterations Maximum times to iterate removal (default = 10) -% p Significance level cutoff (default = 0.01) -% pad FFT padding factor ( -1 corresponds to no padding, -% 0 corresponds to padding to next highest power of 2 -% etc.) (default is 0) -% pnts -% tapers Precomputed tapers from dpss -% taperBandWidth Taper bandwidth (default 2 Hz) -% taperWindowSize Taper sliding window length (default 4 sec) -% taperWindowStep Sliding window step size (default 4 sec = no overlap) -% tau Window overlap smoothing factor (default 100) -% -% This function is based on code originally written by Tim Mullen in a -% package called tmullen-cleanline which is based on the chronux_2 -% libraries. -% - -lineNoiseOut = lineNoiseIn; -%% Remove line frequencies that are greater than Nyquist frequencies -tooLarge = lineNoiseOut.lineFrequencies >= lineNoiseOut.Fs/2; -if any(tooLarge) - warning('cleanLineNoise:LineFrequenciesTooLarge', ... - 'Eliminating frequencies greater than half the sampling rate'); - lineNoiseOut.lineFrequencies(tooLarge) = []; - lineNoiseOut.lineFrequencies = squeeze(lineNoiseOut.lineFrequencies); -end - -%% Set up parameters for blassting the line noise -fRange = lineNoiseOut.fScanBandWidth; -frequencyRanges = repmat(fRange, length(lineNoiseOut.lineFrequencies)); -sRate = lineNoiseOut.Fs; -lineFrequencies = lineNoiseOut.lineFrequencies; -maxIterations = lineNoiseOut.maximumIterations; - -%% Perform the calculation for each channel separately -signal.data = double(signal.data); -chans = lineNoiseOut.lineNoiseChannels; -data = signal.data(chans, :); -parfor ch = 1:size(data, 1) - data(ch, :) = blasst(squeeze(data(ch, :)), lineFrequencies, ... - frequencyRanges, sRate, ... - 'MaximumIterations', maxIterations, ... - 'Verbose', 0); -end -signal.data(chans, :) = data; -clear data; - diff --git a/PrepPipeline/utilities/cleanLineNoise.m b/PrepPipeline/utilities/cleanLineNoise.m index 4c6e48f..f7bcb88 100644 --- a/PrepPipeline/utilities/cleanLineNoise.m +++ b/PrepPipeline/utilities/cleanLineNoise.m @@ -1,63 +1,63 @@ -function [signal, lineNoiseOut] = cleanLineNoise(signal, lineNoiseIn) -% Remove sharp spectral peaks from signal using Sleppian filters -% -% Usage: -% signal = cleanLineNoise(signal) -% [signal, lineNoiseOut] = hcleanLineNoise(signal, lineNoiseIn) -% -% Parameters: -% signal Structure with .data and .srate fields -% lineNoiseIn Input structure with fields described below -% -% Structure parameters (lineNoiseIn): -% fPassBand Frequency band used (default [0, Fs/2] = entire band) -% Fs Sampling frequency -% fScanBandWidth +/- bandwidth centered on each f0 to scan for significant -% lines (TM) -% lineFrequencies Line frequencies to be removed (default -% [60, 120, 180, 240, 300]) -% lineNoiseChannels Channels to remove line noise from (default -% size(data, 1)) -% maximumIterations Maximum times to iterate removal (default = 10) -% p Significance level cutoff (default = 0.01) -% pad FFT padding factor ( -1 corresponds to no padding, -% 0 corresponds to padding to next highest power of 2 -% etc.) (default is 0) -% pnts -% tapers Precomputed tapers from dpss -% taperBandWidth Taper bandwidth (default 2 Hz) -% taperWindowSize Taper sliding window length (default 4 sec) -% taperWindowStep Sliding window step size (default 4 sec = no overlap) -% tau Window overlap smoothing factor (default 100) -% -% This function is based on code originally written by Tim Mullen in a -% package called tmullen-cleanline which is based on the chronux_2 -% libraries. -% - -lineNoiseOut = lineNoiseIn; -%% Remove line frequencies that are greater than Nyquist frequencies -tooLarge = lineNoiseOut.lineFrequencies >= lineNoiseOut.Fs/2; -if any(tooLarge) - warning('cleanLineNoise:LineFrequenciesTooLarge', ... - 'Eliminating frequencies greater than half the sampling rate'); - lineNoiseOut.lineFrequencies(tooLarge) = []; - lineNoiseOut.lineFrequencies = squeeze(lineNoiseOut.lineFrequencies); -end - -%% Set up multi-taper parameters -hbw = lineNoiseOut.taperBandWidth/2; % half-bandwidth -lineNoiseOut.taperTemplate = [hbw, lineNoiseOut.taperWindowSize, 1]; -Nwin = round(lineNoiseOut.Fs*lineNoiseOut.taperWindowSize); % number of samples in window -lineNoiseOut.tapers = checkTapers(lineNoiseOut.taperTemplate, Nwin, lineNoiseOut.Fs); - -%% Perform the calculation for each channel separately -signal.data = double(signal.data); -chans = lineNoiseOut.lineNoiseChannels; -data = signal.data(chans, :); -parfor ch = 1:size(data, 1) - data(ch, :) = removeLinesMovingWindow(squeeze(data(ch, :)), lineNoiseOut); -end -signal.data(chans, :) = data; -clear data; - +function [signal, lineNoiseOut] = cleanLineNoise(signal, lineNoiseIn) +% Remove sharp spectral peaks from signal using Sleppian filters +% +% Usage: +% signal = cleanLineNoise(signal) +% [signal, lineNoiseOut] = hcleanLineNoise(signal, lineNoiseIn) +% +% Parameters: +% signal Structure with .data and .srate fields +% lineNoiseIn Input structure with fields described below +% +% Structure parameters (lineNoiseIn): +% fPassBand Frequency band used (default [0, Fs/2] = entire band) +% Fs Sampling frequency +% fScanBandWidth +/- bandwidth centered on each f0 to scan for significant +% lines (TM) +% lineFrequencies Line frequencies to be removed (default +% [60, 120, 180, 240, 300]) +% lineNoiseChannels Channels to remove line noise from (default +% size(data, 1)) +% maximumIterations Maximum times to iterate removal (default = 10) +% p Significance level cutoff (default = 0.01) +% pad FFT padding factor ( -1 corresponds to no padding, +% 0 corresponds to padding to next highest power of 2 +% etc.) (default is 0) +% pnts +% tapers Precomputed tapers from dpss +% taperBandWidth Taper bandwidth (default 2 Hz) +% taperWindowSize Taper sliding window length (default 4 sec) +% taperWindowStep Sliding window step size (default 4 sec = no overlap) +% tau Window overlap smoothing factor (default 100) +% +% This function is based on code originally written by Tim Mullen in a +% package called tmullen-cleanline which is based on the chronux_2 +% libraries. +% + +lineNoiseOut = lineNoiseIn; +%% Remove line frequencies that are greater than Nyquist frequencies +tooLarge = lineNoiseOut.lineFrequencies >= lineNoiseOut.Fs/2; +if any(tooLarge) + warning('cleanLineNoise:LineFrequenciesTooLarge', ... + 'Eliminating frequencies greater than half the sampling rate'); + lineNoiseOut.lineFrequencies(tooLarge) = []; + lineNoiseOut.lineFrequencies = squeeze(lineNoiseOut.lineFrequencies); +end + +%% Set up multi-taper parameters +hbw = lineNoiseOut.taperBandWidth/2; % half-bandwidth +lineNoiseOut.taperTemplate = [hbw, lineNoiseOut.taperWindowSize, 1]; +Nwin = round(lineNoiseOut.Fs*lineNoiseOut.taperWindowSize); % number of samples in window +lineNoiseOut.tapers = checkTapers(lineNoiseOut.taperTemplate, Nwin, lineNoiseOut.Fs); + +%% Perform the calculation for each channel separately +signal.data = double(signal.data); +chans = lineNoiseOut.lineNoiseChannels; +data = signal.data(chans, :); +parfor ch = 1:size(data, 1) + data(ch, :) = removeLinesMovingWindow(squeeze(data(ch, :)), lineNoiseOut); +end +signal.data(chans, :) = data; +clear data; + diff --git a/PrepPipeline/utilities/findNoisyChannels.m b/PrepPipeline/utilities/findNoisyChannels.m index e91b406..0cdf5b2 100644 --- a/PrepPipeline/utilities/findNoisyChannels.m +++ b/PrepPipeline/utilities/findNoisyChannels.m @@ -1,388 +1,388 @@ -function noisyOut = findNoisyChannels(signal, noisyIn) -% Identify bad channels in EEG using a two-stage approach -% -% reference = findNoisyChannels(signal) -% reference = findNoisyChannels(signal, reference) -% -% First remove bad channels by amplitude, noise level, and correlation -% Apply ransac after these channels have been removed. -% -% Input parameters: -% signal - structure with srate, chanlocs, chaninfo, and data fields -% noisyIn - structure with input parameters -% -% Notes: the signal is assumed to be high-passed. Removing line noise -% is a good idea too. -% -% noisyIn: (fields are filled in on input if not present and propagated to output) -% name - name of the input file -% srate - sample rate in HZ -% samples - number of samples in the data -% evaluationChannels - a vector of channels to use -% channelLocations - a structure of EEG channel locations -% chaninfo - standard EEGLAB chaninfo (nose direction is relevant) -% chanlocs - standard EEGLAB chanlocs structure -% robustDeviationThreshold - z score cutoff for robust channel deviation -% highFrequencyNoiseThreshold - z score cutoff for SNR (signal above 50 Hz) -% correlationWindowSeconds - correlation window size in seconds (default = 1 sec) -% correlationThreshold - correlation below which window is bad (default = 0.4) -% badTimeThreshold - cutoff fraction of bad corr windows (default = 0.01) -% ransacSampleSize - samples for computing ransac (default = 50) -% ransacChannelFraction - fraction of channels for robust reconstruction (default = 0.25) -% ransacCorrelationThreshold - cutoff correlation for abnormal wrt neighbors(default = 0.75) -% ransacUnbrokenTime - cutoff fraction of time channel can have poor ransac predictability (default = 0.4) -% ransacWindowSeconds - correlation window for ransac (default = 5 sec) -% -% Output parameters (c channels, w windows): -% ransacPerformed - true if there were enough good channels to do ransac -% noisyChannels - list of identified bad channel numbers -% badChannelsFromCorrelation - list of bad channels identified by correlation -% badChannelsFromDeviation - list of bad channels identified by amplitude -% badChannelsFromHFNoise - list of bad channels identified by SNR -% badChannelsFromRansac - list of channels identified by ransac -% fractionBadCorrelationWindows - c x 1 vector with fraction of bad correlation windows -% robustChannelDeviation - c x 1 vector with robust measure of average channel deviation -% zscoreHFNoise - c x 1 vector with measure of channel noise level -% maximumCorrelations - w x c array with max window correlation -% ransacCorrelations = c x wr array with ransac correlations -% -% This function uses 4 methods for detecting bad channels after removing -% from consideration channels that have NaN data or channels that are -% identically constant. -% -% Method 1: too low or high amplitude. If the z score of robust -% channel deviation falls below robustDeviationThreshold, the channel is -% considered to be bad. -% Method 2: too low an SNR. If the z score of estimate of signal above -% 50 Hz to that below 50 Hz above highFrequencyNoiseThreshold, the channel -% is considered to be bad. -% -% Method 3: low correlation with other channels. Here correlationWindowSize is the window -% size over which the correlation is computed. If the maximum -% correlation of the channel to the other channels falls below -% correlationThreshold, the channel is considered bad in that window. -% If the fraction of bad correlation windows for a channel -% exceeds badTimeThreshold, the channel is marked as bad. -% -% After the channels from methods 2 and 3 are removed, method 4 is -% computed on the remaining signals -% -% Method 4: each channel is predicted using ransac interpolation based -% on a ransac fraction of the channels. If the correlation of -% the prediction to the actual behavior is too low for too -% long, the channel is marked as bad. -% -% Assumptions: -% - The signal is a structure of continuous data with data, srate, chanlocs, -% and chaninfo fields. -% - The signal.data has been high pass filtered. -% - No segments of the EEG data have been removed - -% Methods 1 and 4 are adapted from code by Christian Kothe and Methods 2 -% and 3 are adapted from code by Nima Bigdely-Shamlo -% -%% Check the incoming parameters -if nargin < 1 - error('findNoisyChannels:NotEnoughArguments', 'requires at least 1 argument'); -elseif isstruct(signal) && ~isfield(signal, 'data') - error('findNoisyChannels:NoDataField', 'requires a structure data field'); -elseif size(signal.data, 3) ~= 1 - error('findNoisyChannels:DataNotContinuous', 'data must be a 2D array'); -elseif nargin < 2 || ~exist('noisyIn', 'var') || isempty(noisyIn) - noisyIn = struct(); -end - -%% Set the defaults and initialize as needed -noisyOut = getNoisyStructure(); -defaults = getPrepDefaults(signal, 'reference'); -[noisyOut, errors] = checkPrepDefaults(noisyIn, noisyOut, defaults); -if ~isempty(errors) - error('findNoisyChannels:BadParameters', ['|' sprintf('%s|', errors{:})]); -end -%% Fix the channel locations -channelLocations = noisyOut.channelLocations; -evaluationChannels = sort(noisyOut.evaluationChannels); % Make sure channels are sorted -evaluationChannels = evaluationChannels(:)'; % Make sure row vector -noisyOut.evaluationChannels = evaluationChannels; -originalChannels = 1:size(signal.data, 1); - -%% Extract the data required -data = signal.data; -originalNumberChannels = size(data, 1); % Save the original channels -data = double(data(evaluationChannels, :))'; % Remove the unneeded channels -signalSize = size(data, 1); -correlationFrames = noisyOut.correlationWindowSeconds * signal.srate; -correlationWindow = 0:(correlationFrames - 1); -correlationOffsets = 1:correlationFrames:(signalSize-correlationFrames); -WCorrelation = length(correlationOffsets); -ransacFrames = noisyOut.ransacWindowSeconds*noisyOut.srate; -ransacWindow = 0:(ransacFrames - 1); -ransacOffsets = 1:ransacFrames:(signalSize-ransacFrames); -WRansac = length(ransacOffsets); -noisyOut.zscoreHFNoise = zeros(originalNumberChannels, 1); -noisyOut.noiseLevels = zeros(originalNumberChannels, WCorrelation); -noisyOut.maximumCorrelations = ones(originalNumberChannels, WCorrelation); -noisyOut.dropOuts = zeros(originalNumberChannels, WCorrelation); -noisyOut.correlationOffsets = correlationOffsets; -noisyOut.channelDeviations = zeros(originalNumberChannels, WCorrelation); -noisyOut.robustChannelDeviation = zeros(originalNumberChannels, 1); -noisyOut.ransacCorrelations = ones(originalNumberChannels, WRansac); -noisyOut.ransacOffsets = ransacOffsets; - -%% Detect constant or NaN channels and remove from consideration -nanChannelMask = sum(isnan(data), 1) > 0; -noSignalChannelMask = mad(data, 1, 1) < 10e-10 | std(data, 1, 1) < 10e-10; -noisyOut.noisyChannels.badChannelsFromNaNs = evaluationChannels(nanChannelMask); -noisyOut.noisyChannels.badChannelsFromNoData = evaluationChannels(noSignalChannelMask); -evaluationChannels = setdiff(evaluationChannels, ... - union(noisyOut.noisyChannels.badChannelsFromNaNs, ... - noisyOut.noisyChannels.badChannelsFromNoData)); -data = signal.data; -data = double(data(evaluationChannels, :))'; -[signalSize, numberChannels] = size(data); - -%% Method 1: Unusually high or low amplitude (using robust std) -channelDeviation = 0.7413 *iqr(data); % Robust estimate of SD -channelDeviationSD = 0.7413 * iqr(channelDeviation); -channelDeviationMedian = nanmedian(channelDeviation); -noisyOut.robustChannelDeviation(evaluationChannels) = ... - (channelDeviation - channelDeviationMedian) / channelDeviationSD; - -% Find channels with unusually high deviation -badChannelsFromDeviation = ... - abs(noisyOut.robustChannelDeviation) > ... - noisyOut.robustDeviationThreshold | ... - isnan(noisyOut.robustChannelDeviation); -badChannelsFromDeviation = originalChannels(badChannelsFromDeviation); -noisyOut.noisyChannels.badChannelsFromDeviation = badChannelsFromDeviation(:)'; -noisyOut.channelDeviationMedian = channelDeviationMedian; -noisyOut.channelDeviationSD = channelDeviationSD; - -%% Method 2: Compute the SNR (based on Christian Kothe's clean_channels) -% Note: RANSAC uses the filtered values X of the data -if noisyOut.srate > 100 - % Remove signal content above 50Hz and below 1 Hz - B = design_fir(100,[2*[0 45 50]/noisyOut.srate 1],[1 1 0 0]); - X = zeros(signalSize, numberChannels); - parfor k = 1:numberChannels % Could be changed to parfor - X(:,k) = filtfilt_fast(B, 1, data(:, k)); end - % Determine z-scored level of EM noise-to-signal ratio for each channel - noisiness = mad(data- X, 1)./mad(X, 1); - noisinessMedian = nanmedian(noisiness); - noisinessSD = mad(noisiness, 1)*1.4826; - zscoreHFNoiseTemp = (noisiness - noisinessMedian) ./ noisinessSD; - noiseMask = (zscoreHFNoiseTemp > noisyOut.highFrequencyNoiseThreshold) | ... - isnan(zscoreHFNoiseTemp); - % Remap channels to original numbering - badChannelsFromHFNoise = evaluationChannels(noiseMask); - noisyOut.noisyChannels.badChannelsFromHFNoise = badChannelsFromHFNoise(:)'; -else - X = data; - noisinessMedian = 0; - noisinessSD = 1; - zscoreHFNoiseTemp = zeros(numberChannels, 1); - noisyOut.noisyChannels.badChannelsFromHFNoise = []; -end - -% Remap the channels to original numbering for the zscoreHFNoise -noisyOut.zscoreHFNoise(evaluationChannels) = zscoreHFNoiseTemp; -noisyOut.noisinessMedian = noisinessMedian; -noisyOut.noisinessSD = noisinessSD; - -%% Method 3: Global correlation criteria (from Nima Bigdely-Shamlo) -channelCorrelations = ones(WCorrelation, numberChannels); -noiseLevels = zeros(WCorrelation, numberChannels); -channelDeviations = zeros(WCorrelation, numberChannels); -n = length(correlationWindow); -xWin = reshape(X(1:n*WCorrelation, :)', numberChannels, n, WCorrelation); -dataWin = reshape(data(1:n*WCorrelation, :)', numberChannels, n, WCorrelation); -parfor k = 1:WCorrelation - eegPortion = squeeze(xWin(:, :, k))'; - dataPortion = squeeze(dataWin(:, :, k))'; - windowCorrelation = corrcoef(eegPortion); - abs_corr = abs(windowCorrelation - diag(diag(windowCorrelation))); - channelCorrelations(k, :) = quantile(abs_corr, 0.98); - noiseLevels(k, :) = mad(dataPortion - eegPortion, 1)./mad(eegPortion, 1); - channelDeviations(k, :) = 0.7413 *iqr(dataPortion); -end -dropOuts = isnan(channelCorrelations) | isnan(noiseLevels); -channelCorrelations(dropOuts) = 0.0; -noiseLevels(dropOuts) = 0.0; -clear xWin; -clear dataWin; -noisyOut.maximumCorrelations(evaluationChannels, :) = channelCorrelations'; -noisyOut.noiseLevels(evaluationChannels, :) = noiseLevels'; -noisyOut.channelDeviations(evaluationChannels, :) = channelDeviations'; -noisyOut.dropOuts(evaluationChannels, :) = dropOuts'; -thresholdedCorrelations = ... - noisyOut.maximumCorrelations < noisyOut.correlationThreshold; -fractionBadCorrelationWindows = mean(thresholdedCorrelations, 2); -fractionBadDropOutWindows = mean(noisyOut.dropOuts, 2); - -% Remap channels to their original numbers -badChannelsFromCorrelation = find(fractionBadCorrelationWindows > noisyOut.badTimeThreshold); -noisyOut.noisyChannels.badChannelsFromCorrelation = badChannelsFromCorrelation(:)'; -badChannelsFromDropOuts = find(fractionBadDropOutWindows > noisyOut.badTimeThreshold); -noisyOut.noisyChannels.badChannelsFromDropOuts = badChannelsFromDropOuts(:)'; -noisyOut.medianMaxCorrelation = median(noisyOut.maximumCorrelations, 2); - -%% Bad so far by amplitude and correlation (take these out before doing ransac) -noisyChannels = union(noisyOut.noisyChannels.badChannelsFromDeviation, ... - union(noisyOut.noisyChannels.badChannelsFromCorrelation, ... - noisyOut.noisyChannels.badChannelsFromDropOuts)); - -%% Method 4: Ransac corelation (may not be performed) -% Setup for ransac (if a 2-stage algorithm, remove other bad channels first) -if noisyOut.ransacOff - noisyOut.ransacBadWindowFraction = 0; - noisyOut.ransacPerformed = false; -elseif isempty(channelLocations) - warning('findNoisyChannels:noChannelLocation', ... - 'ransac could not be computed because there were no channel locations'); - noisyOut.ransacBadWindowFraction = 0; - noisyOut.ransacPerformed = false; -else % Set up parameters and make sure enough good channels to proceed - [ransacChannels, idiff] = setdiff(evaluationChannels, noisyChannels); - X = X(:, idiff); - - % Calculate the parameters for ransac - ransacSubset = round(noisyOut.ransacChannelFraction*size(data, 2)); - if noisyOut.ransacUnbrokenTime < 0 - error('find_noisyChannels:BadUnbrokenParameter', ... - 'ransacUnbrokenTime must be greater than 0'); - elseif noisyOut.ransacUnbrokenTime < 1 - ransacUnbrokenFrames = signalSize*noisyOut.ransacUnbrokenTime; - else - ransacUnbrokenFrames = srate*noisyOut.ransacUnbrokenTime; - end - - nchanlocs = channelLocations(ransacChannels); - if length(nchanlocs) ~= size(nchanlocs, 2) - nchanlocs = nchanlocs'; - end - if length(nchanlocs) < ransacSubset + 1 || length(nchanlocs) < 3 || ... - ransacSubset < 2 - warning('find_noisyChannels:NotEnoughGoodChannels', ... - 'Too many channels have failed quality tests to perform ransac'); - noisyOut.ransacBadWindowFraction = 0; - noisyOut.ransacPerformed = false; - end -end - -if noisyOut.ransacPerformed - try - % Calculate all-channel reconstruction matrices from random channel subsets - locs = [cell2mat({nchanlocs.X}); cell2mat({nchanlocs.Y});cell2mat({nchanlocs.Z})]; - catch err - error('findNoisyChannels:NoXYZChannelLocations', ... - 'Must provide valid channel locations'); - end - if isempty(locs) || size(locs, 2) ~= length(ransacChannels) ... - || any(isnan(locs(:))) - error('find_noisyChannels:EmptyChannelLocations', ... - 'The signal chanlocs must have valid X, Y, and Z components'); - end - P = hlp_microcache('cleanchans', @calc_projector, locs, ... - noisyOut.ransacSampleSize, ransacSubset); - ransacCorrelationsT = zeros(length(locs), WRansac); - - % Calculate each channel's correlation to its RANSAC reconstruction for each window - n = length(ransacWindow); - m = length(ransacChannels); - p = noisyOut.ransacSampleSize; - Xwin = reshape(X(1:n*WRansac, :)', m, n, WRansac); - parfor k = 1:WRansac - ransacCorrelationsT(:, k) = ... - calculateRansacWindow(squeeze(Xwin(:, :, k))', P, n, m, p); - end - clear Xwin; - noisyOut.ransacCorrelations(ransacChannels, :) = ransacCorrelationsT; - flagged = noisyOut.ransacCorrelations < noisyOut.ransacCorrelationThreshold; - badChannelsFromRansac = find(sum(flagged, 2)*ransacFrames > ransacUnbrokenFrames)'; - noisyOut.noisyChannels.badChannelsFromRansac = badChannelsFromRansac(:)'; - noisyOut.ransacBadWindowFraction = sum(flagged, 2)/size(flagged, 2); -end - -% Combine bad channels detected from all methods -noisy = noisyOut.noisyChannels; -noisyOut.noisyChannels.badChannelsFromLowSNR = ... - intersect(noisy.badChannelsFromHFNoise, noisy.badChannelsFromCorrelation); -noisyChannels = union(noisyChannels, ... - union(union(noisy.badChannelsFromRansac, ... - noisy.badChannelsFromHFNoise), ... - union(noisy.badChannelsFromNaNs, ... - noisy.badChannelsFromNoData))); -noisyOut.noisyChannels.all = noisyChannels(:)'; -noisyOut.medianMaxCorrelation = median(noisyOut.maximumCorrelations, 2); - -%% Helper functions for findNoisyChannels -function P = calc_projector(locs, numberSamples, subsetSize) -% Calculate a bag of reconstruction matrices from random channel subsets - -[permutedLocations, subsets] = getRandomSubsets(locs, subsetSize, numberSamples); -randomSamples = cell(1, numberSamples); -parfor k = 1:numberSamples - tmp = zeros(size(locs, 2)); - slice = subsets(k, :); - tmp(slice, :) = real(spherical_interpolate(permutedLocations(:, :, k), locs))'; - randomSamples{k} = tmp; -end -P = horzcat(randomSamples{:}); - -function [permutedLocations, subsets] = getRandomSubsets(locs, subsetSize, numberSamples) - stream = RandStream('mt19937ar', 'Seed', 435656); - numberChannels = size(locs, 2); - permutedLocations = zeros(3, subsetSize, numberSamples); - subsets = zeros(numberSamples, subsetSize); - for k = 1:numberSamples - subset = randsample(1:numberChannels, subsetSize, stream); - subsets(k, :) = subset; - permutedLocations(:, :, k) = locs(:, subset); - end - -function Y = randsample(X, num, stream) -Y = zeros(1, num); -for k = 1:num - pick = round(1 + (length(X)-1).*rand(stream)); - Y(k) = X(pick); - X(pick) = []; -end - -function rX = calculateRansacWindow(XX, P, n, m, p) - YY = sort(reshape(XX*P, n, m, p),3); - YY = YY(:, :, round(end/2)); - rX = sum(XX.*YY)./(sqrt(sum(XX.^2)).*sqrt(sum(YY.^2))); - -function noisyOut = getNoisyStructure() - noisyOut = struct('srate', [], ... - 'samples', [], ... - 'evaluationChannels', [], ... - 'channelLocations', [], ... - 'robustDeviationThreshold', [], ... - 'highFrequencyNoiseThreshold', [], ... - 'correlationWindowSeconds', [], ... - 'correlationThreshold', [], ... - 'badTimeThreshold', [], ... - 'ransacSampleSize', [], ... - 'ransacChannelFraction', [], ... - 'ransacCorrelationThreshold', [], ... - 'ransacUnbrokenTime', [], ... - 'ransacWindowSeconds', [], ... - 'noisyChannels', getBadChannelStructure(), ... - 'ransacPerformed', true, ... - 'channelDeviationMedian', [], ... - 'channelDeviationSD', [], ... - 'channelDeviations', [], ... - 'robustChannelDeviation', [], ... - 'noisinessMedian', [], ... - 'noisinessSD', [], ... - 'zscoreHFNoise', [], ... - 'noiseLevels', [], ... - 'maximumCorrelations', [], ... - 'dropOuts', [], ... - 'medianMaxCorrelation', [], ... - 'correlationOffsets', [], ... - 'ransacCorrelations', [], ... - 'ransacOffsets', [], ... - 'ransacBadWindowFraction', []); - +function noisyOut = findNoisyChannels(signal, noisyIn) +% Identify bad channels in EEG using a two-stage approach +% +% reference = findNoisyChannels(signal) +% reference = findNoisyChannels(signal, reference) +% +% First remove bad channels by amplitude, noise level, and correlation +% Apply ransac after these channels have been removed. +% +% Input parameters: +% signal - structure with srate, chanlocs, chaninfo, and data fields +% noisyIn - structure with input parameters +% +% Notes: the signal is assumed to be high-passed. Removing line noise +% is a good idea too. +% +% noisyIn: (fields are filled in on input if not present and propagated to output) +% name - name of the input file +% srate - sample rate in HZ +% samples - number of samples in the data +% evaluationChannels - a vector of channels to use +% channelLocations - a structure of EEG channel locations +% chaninfo - standard EEGLAB chaninfo (nose direction is relevant) +% chanlocs - standard EEGLAB chanlocs structure +% robustDeviationThreshold - z score cutoff for robust channel deviation +% highFrequencyNoiseThreshold - z score cutoff for SNR (signal above 50 Hz) +% correlationWindowSeconds - correlation window size in seconds (default = 1 sec) +% correlationThreshold - correlation below which window is bad (default = 0.4) +% badTimeThreshold - cutoff fraction of bad corr windows (default = 0.01) +% ransacSampleSize - samples for computing ransac (default = 50) +% ransacChannelFraction - fraction of channels for robust reconstruction (default = 0.25) +% ransacCorrelationThreshold - cutoff correlation for abnormal wrt neighbors(default = 0.75) +% ransacUnbrokenTime - cutoff fraction of time channel can have poor ransac predictability (default = 0.4) +% ransacWindowSeconds - correlation window for ransac (default = 5 sec) +% +% Output parameters (c channels, w windows): +% ransacPerformed - true if there were enough good channels to do ransac +% noisyChannels - list of identified bad channel numbers +% badChannelsFromCorrelation - list of bad channels identified by correlation +% badChannelsFromDeviation - list of bad channels identified by amplitude +% badChannelsFromHFNoise - list of bad channels identified by SNR +% badChannelsFromRansac - list of channels identified by ransac +% fractionBadCorrelationWindows - c x 1 vector with fraction of bad correlation windows +% robustChannelDeviation - c x 1 vector with robust measure of average channel deviation +% zscoreHFNoise - c x 1 vector with measure of channel noise level +% maximumCorrelations - w x c array with max window correlation +% ransacCorrelations = c x wr array with ransac correlations +% +% This function uses 4 methods for detecting bad channels after removing +% from consideration channels that have NaN data or channels that are +% identically constant. +% +% Method 1: too low or high amplitude. If the z score of robust +% channel deviation falls below robustDeviationThreshold, the channel is +% considered to be bad. +% Method 2: too low an SNR. If the z score of estimate of signal above +% 50 Hz to that below 50 Hz above highFrequencyNoiseThreshold, the channel +% is considered to be bad. +% +% Method 3: low correlation with other channels. Here correlationWindowSize is the window +% size over which the correlation is computed. If the maximum +% correlation of the channel to the other channels falls below +% correlationThreshold, the channel is considered bad in that window. +% If the fraction of bad correlation windows for a channel +% exceeds badTimeThreshold, the channel is marked as bad. +% +% After the channels from methods 2 and 3 are removed, method 4 is +% computed on the remaining signals +% +% Method 4: each channel is predicted using ransac interpolation based +% on a ransac fraction of the channels. If the correlation of +% the prediction to the actual behavior is too low for too +% long, the channel is marked as bad. +% +% Assumptions: +% - The signal is a structure of continuous data with data, srate, chanlocs, +% and chaninfo fields. +% - The signal.data has been high pass filtered. +% - No segments of the EEG data have been removed + +% Methods 1 and 4 are adapted from code by Christian Kothe and Methods 2 +% and 3 are adapted from code by Nima Bigdely-Shamlo +% +%% Check the incoming parameters +if nargin < 1 + error('findNoisyChannels:NotEnoughArguments', 'requires at least 1 argument'); +elseif isstruct(signal) && ~isfield(signal, 'data') + error('findNoisyChannels:NoDataField', 'requires a structure data field'); +elseif size(signal.data, 3) ~= 1 + error('findNoisyChannels:DataNotContinuous', 'data must be a 2D array'); +elseif nargin < 2 || ~exist('noisyIn', 'var') || isempty(noisyIn) + noisyIn = struct(); +end + +%% Set the defaults and initialize as needed +noisyOut = getNoisyStructure(); +defaults = getPrepDefaults(signal, 'reference'); +[noisyOut, errors] = checkPrepDefaults(noisyIn, noisyOut, defaults); +if ~isempty(errors) + error('findNoisyChannels:BadParameters', ['|' sprintf('%s|', errors{:})]); +end +%% Fix the channel locations +channelLocations = noisyOut.channelLocations; +evaluationChannels = sort(noisyOut.evaluationChannels); % Make sure channels are sorted +evaluationChannels = evaluationChannels(:)'; % Make sure row vector +noisyOut.evaluationChannels = evaluationChannels; +originalChannels = 1:size(signal.data, 1); + +%% Extract the data required +data = signal.data; +originalNumberChannels = size(data, 1); % Save the original channels +data = double(data(evaluationChannels, :))'; % Remove the unneeded channels +signalSize = size(data, 1); +correlationFrames = noisyOut.correlationWindowSeconds * signal.srate; +correlationWindow = 0:(correlationFrames - 1); +correlationOffsets = 1:correlationFrames:(signalSize-correlationFrames); +WCorrelation = length(correlationOffsets); +ransacFrames = noisyOut.ransacWindowSeconds*noisyOut.srate; +ransacWindow = 0:(ransacFrames - 1); +ransacOffsets = 1:ransacFrames:(signalSize-ransacFrames); +WRansac = length(ransacOffsets); +noisyOut.zscoreHFNoise = zeros(originalNumberChannels, 1); +noisyOut.noiseLevels = zeros(originalNumberChannels, WCorrelation); +noisyOut.maximumCorrelations = ones(originalNumberChannels, WCorrelation); +noisyOut.dropOuts = zeros(originalNumberChannels, WCorrelation); +noisyOut.correlationOffsets = correlationOffsets; +noisyOut.channelDeviations = zeros(originalNumberChannels, WCorrelation); +noisyOut.robustChannelDeviation = zeros(originalNumberChannels, 1); +noisyOut.ransacCorrelations = ones(originalNumberChannels, WRansac); +noisyOut.ransacOffsets = ransacOffsets; + +%% Detect constant or NaN channels and remove from consideration +nanChannelMask = sum(isnan(data), 1) > 0; +noSignalChannelMask = mad(data, 1, 1) < 10e-10 | std(data, 1, 1) < 10e-10; +noisyOut.noisyChannels.badChannelsFromNaNs = evaluationChannels(nanChannelMask); +noisyOut.noisyChannels.badChannelsFromNoData = evaluationChannels(noSignalChannelMask); +evaluationChannels = setdiff(evaluationChannels, ... + union(noisyOut.noisyChannels.badChannelsFromNaNs, ... + noisyOut.noisyChannels.badChannelsFromNoData)); +data = signal.data; +data = double(data(evaluationChannels, :))'; +[signalSize, numberChannels] = size(data); + +%% Method 1: Unusually high or low amplitude (using robust std) +channelDeviation = 0.7413 *iqr(data); % Robust estimate of SD +channelDeviationSD = 0.7413 * iqr(channelDeviation); +channelDeviationMedian = nanmedian(channelDeviation); +noisyOut.robustChannelDeviation(evaluationChannels) = ... + (channelDeviation - channelDeviationMedian) / channelDeviationSD; + +% Find channels with unusually high deviation +badChannelsFromDeviation = ... + abs(noisyOut.robustChannelDeviation) > ... + noisyOut.robustDeviationThreshold | ... + isnan(noisyOut.robustChannelDeviation); +badChannelsFromDeviation = originalChannels(badChannelsFromDeviation); +noisyOut.noisyChannels.badChannelsFromDeviation = badChannelsFromDeviation(:)'; +noisyOut.channelDeviationMedian = channelDeviationMedian; +noisyOut.channelDeviationSD = channelDeviationSD; + +%% Method 2: Compute the SNR (based on Christian Kothe's clean_channels) +% Note: RANSAC uses the filtered values X of the data +if noisyOut.srate > 100 + % Remove signal content above 50Hz and below 1 Hz + B = design_fir(100,[2*[0 45 50]/noisyOut.srate 1],[1 1 0 0]); + X = zeros(signalSize, numberChannels); + parfor k = 1:numberChannels % Could be changed to parfor + X(:,k) = filtfilt_fast(B, 1, data(:, k)); end + % Determine z-scored level of EM noise-to-signal ratio for each channel + noisiness = mad(data- X, 1)./mad(X, 1); + noisinessMedian = nanmedian(noisiness); + noisinessSD = mad(noisiness, 1)*1.4826; + zscoreHFNoiseTemp = (noisiness - noisinessMedian) ./ noisinessSD; + noiseMask = (zscoreHFNoiseTemp > noisyOut.highFrequencyNoiseThreshold) | ... + isnan(zscoreHFNoiseTemp); + % Remap channels to original numbering + badChannelsFromHFNoise = evaluationChannels(noiseMask); + noisyOut.noisyChannels.badChannelsFromHFNoise = badChannelsFromHFNoise(:)'; +else + X = data; + noisinessMedian = 0; + noisinessSD = 1; + zscoreHFNoiseTemp = zeros(numberChannels, 1); + noisyOut.noisyChannels.badChannelsFromHFNoise = []; +end + +% Remap the channels to original numbering for the zscoreHFNoise +noisyOut.zscoreHFNoise(evaluationChannels) = zscoreHFNoiseTemp; +noisyOut.noisinessMedian = noisinessMedian; +noisyOut.noisinessSD = noisinessSD; + +%% Method 3: Global correlation criteria (from Nima Bigdely-Shamlo) +channelCorrelations = ones(WCorrelation, numberChannels); +noiseLevels = zeros(WCorrelation, numberChannels); +channelDeviations = zeros(WCorrelation, numberChannels); +n = length(correlationWindow); +xWin = reshape(X(1:n*WCorrelation, :)', numberChannels, n, WCorrelation); +dataWin = reshape(data(1:n*WCorrelation, :)', numberChannels, n, WCorrelation); +parfor k = 1:WCorrelation + eegPortion = squeeze(xWin(:, :, k))'; + dataPortion = squeeze(dataWin(:, :, k))'; + windowCorrelation = corrcoef(eegPortion); + abs_corr = abs(windowCorrelation - diag(diag(windowCorrelation))); + channelCorrelations(k, :) = quantile(abs_corr, 0.98); + noiseLevels(k, :) = mad(dataPortion - eegPortion, 1)./mad(eegPortion, 1); + channelDeviations(k, :) = 0.7413 *iqr(dataPortion); +end +dropOuts = isnan(channelCorrelations) | isnan(noiseLevels); +channelCorrelations(dropOuts) = 0.0; +noiseLevels(dropOuts) = 0.0; +clear xWin; +clear dataWin; +noisyOut.maximumCorrelations(evaluationChannels, :) = channelCorrelations'; +noisyOut.noiseLevels(evaluationChannels, :) = noiseLevels'; +noisyOut.channelDeviations(evaluationChannels, :) = channelDeviations'; +noisyOut.dropOuts(evaluationChannels, :) = dropOuts'; +thresholdedCorrelations = ... + noisyOut.maximumCorrelations < noisyOut.correlationThreshold; +fractionBadCorrelationWindows = mean(thresholdedCorrelations, 2); +fractionBadDropOutWindows = mean(noisyOut.dropOuts, 2); + +% Remap channels to their original numbers +badChannelsFromCorrelation = find(fractionBadCorrelationWindows > noisyOut.badTimeThreshold); +noisyOut.noisyChannels.badChannelsFromCorrelation = badChannelsFromCorrelation(:)'; +badChannelsFromDropOuts = find(fractionBadDropOutWindows > noisyOut.badTimeThreshold); +noisyOut.noisyChannels.badChannelsFromDropOuts = badChannelsFromDropOuts(:)'; +noisyOut.medianMaxCorrelation = median(noisyOut.maximumCorrelations, 2); + +%% Bad so far by amplitude and correlation (take these out before doing ransac) +noisyChannels = union(noisyOut.noisyChannels.badChannelsFromDeviation, ... + union(noisyOut.noisyChannels.badChannelsFromCorrelation, ... + noisyOut.noisyChannels.badChannelsFromDropOuts)); + +%% Method 4: Ransac corelation (may not be performed) +% Setup for ransac (if a 2-stage algorithm, remove other bad channels first) +if noisyOut.ransacOff + noisyOut.ransacBadWindowFraction = 0; + noisyOut.ransacPerformed = false; +elseif isempty(channelLocations) + warning('findNoisyChannels:noChannelLocation', ... + 'ransac could not be computed because there were no channel locations'); + noisyOut.ransacBadWindowFraction = 0; + noisyOut.ransacPerformed = false; +else % Set up parameters and make sure enough good channels to proceed + [ransacChannels, idiff] = setdiff(evaluationChannels, noisyChannels); + X = X(:, idiff); + + % Calculate the parameters for ransac + ransacSubset = round(noisyOut.ransacChannelFraction*size(data, 2)); + if noisyOut.ransacUnbrokenTime < 0 + error('find_noisyChannels:BadUnbrokenParameter', ... + 'ransacUnbrokenTime must be greater than 0'); + elseif noisyOut.ransacUnbrokenTime < 1 + ransacUnbrokenFrames = signalSize*noisyOut.ransacUnbrokenTime; + else + ransacUnbrokenFrames = srate*noisyOut.ransacUnbrokenTime; + end + + nchanlocs = channelLocations(ransacChannels); + if length(nchanlocs) ~= size(nchanlocs, 2) + nchanlocs = nchanlocs'; + end + if length(nchanlocs) < ransacSubset + 1 || length(nchanlocs) < 3 || ... + ransacSubset < 2 + warning('find_noisyChannels:NotEnoughGoodChannels', ... + 'Too many channels have failed quality tests to perform ransac'); + noisyOut.ransacBadWindowFraction = 0; + noisyOut.ransacPerformed = false; + end +end + +if noisyOut.ransacPerformed + try + % Calculate all-channel reconstruction matrices from random channel subsets + locs = [cell2mat({nchanlocs.X}); cell2mat({nchanlocs.Y});cell2mat({nchanlocs.Z})]; + catch err + error('findNoisyChannels:NoXYZChannelLocations', ... + 'Must provide valid channel locations'); + end + if isempty(locs) || size(locs, 2) ~= length(ransacChannels) ... + || any(isnan(locs(:))) + error('find_noisyChannels:EmptyChannelLocations', ... + 'The signal chanlocs must have valid X, Y, and Z components'); + end + P = hlp_microcache('cleanchans', @calc_projector, locs, ... + noisyOut.ransacSampleSize, ransacSubset); + ransacCorrelationsT = zeros(length(locs), WRansac); + + % Calculate each channel's correlation to its RANSAC reconstruction for each window + n = length(ransacWindow); + m = length(ransacChannels); + p = noisyOut.ransacSampleSize; + Xwin = reshape(X(1:n*WRansac, :)', m, n, WRansac); + parfor k = 1:WRansac + ransacCorrelationsT(:, k) = ... + calculateRansacWindow(squeeze(Xwin(:, :, k))', P, n, m, p); + end + clear Xwin; + noisyOut.ransacCorrelations(ransacChannels, :) = ransacCorrelationsT; + flagged = noisyOut.ransacCorrelations < noisyOut.ransacCorrelationThreshold; + badChannelsFromRansac = find(sum(flagged, 2)*ransacFrames > ransacUnbrokenFrames)'; + noisyOut.noisyChannels.badChannelsFromRansac = badChannelsFromRansac(:)'; + noisyOut.ransacBadWindowFraction = sum(flagged, 2)/size(flagged, 2); +end + +% Combine bad channels detected from all methods +noisy = noisyOut.noisyChannels; +noisyOut.noisyChannels.badChannelsFromLowSNR = ... + intersect(noisy.badChannelsFromHFNoise, noisy.badChannelsFromCorrelation); +noisyChannels = union(noisyChannels, ... + union(union(noisy.badChannelsFromRansac, ... + noisy.badChannelsFromHFNoise), ... + union(noisy.badChannelsFromNaNs, ... + noisy.badChannelsFromNoData))); +noisyOut.noisyChannels.all = noisyChannels(:)'; +noisyOut.medianMaxCorrelation = median(noisyOut.maximumCorrelations, 2); + +%% Helper functions for findNoisyChannels +function P = calc_projector(locs, numberSamples, subsetSize) +% Calculate a bag of reconstruction matrices from random channel subsets + +[permutedLocations, subsets] = getRandomSubsets(locs, subsetSize, numberSamples); +randomSamples = cell(1, numberSamples); +parfor k = 1:numberSamples + tmp = zeros(size(locs, 2)); + slice = subsets(k, :); + tmp(slice, :) = real(spherical_interpolate(permutedLocations(:, :, k), locs))'; + randomSamples{k} = tmp; +end +P = horzcat(randomSamples{:}); + +function [permutedLocations, subsets] = getRandomSubsets(locs, subsetSize, numberSamples) + stream = RandStream('mt19937ar', 'Seed', 435656); + numberChannels = size(locs, 2); + permutedLocations = zeros(3, subsetSize, numberSamples); + subsets = zeros(numberSamples, subsetSize); + for k = 1:numberSamples + subset = randsample(1:numberChannels, subsetSize, stream); + subsets(k, :) = subset; + permutedLocations(:, :, k) = locs(:, subset); + end + +function Y = randsample(X, num, stream) +Y = zeros(1, num); +for k = 1:num + pick = round(1 + (length(X)-1).*rand(stream)); + Y(k) = X(pick); + X(pick) = []; +end + +function rX = calculateRansacWindow(XX, P, n, m, p) + YY = sort(reshape(XX*P, n, m, p),3); + YY = YY(:, :, round(end/2)); + rX = sum(XX.*YY)./(sqrt(sum(XX.^2)).*sqrt(sum(YY.^2))); + +function noisyOut = getNoisyStructure() + noisyOut = struct('srate', [], ... + 'samples', [], ... + 'evaluationChannels', [], ... + 'channelLocations', [], ... + 'robustDeviationThreshold', [], ... + 'highFrequencyNoiseThreshold', [], ... + 'correlationWindowSeconds', [], ... + 'correlationThreshold', [], ... + 'badTimeThreshold', [], ... + 'ransacSampleSize', [], ... + 'ransacChannelFraction', [], ... + 'ransacCorrelationThreshold', [], ... + 'ransacUnbrokenTime', [], ... + 'ransacWindowSeconds', [], ... + 'noisyChannels', getBadChannelStructure(), ... + 'ransacPerformed', true, ... + 'channelDeviationMedian', [], ... + 'channelDeviationSD', [], ... + 'channelDeviations', [], ... + 'robustChannelDeviation', [], ... + 'noisinessMedian', [], ... + 'noisinessSD', [], ... + 'zscoreHFNoise', [], ... + 'noiseLevels', [], ... + 'maximumCorrelations', [], ... + 'dropOuts', [], ... + 'medianMaxCorrelation', [], ... + 'correlationOffsets', [], ... + 'ransacCorrelations', [], ... + 'ransacOffsets', [], ... + 'ransacBadWindowFraction', []); + diff --git a/PrepPipeline/utilities/getPrepDefaults.m b/PrepPipeline/utilities/getPrepDefaults.m index 70cb4a6..2805116 100644 --- a/PrepPipeline/utilities/getPrepDefaults.m +++ b/PrepPipeline/utilities/getPrepDefaults.m @@ -93,7 +93,7 @@ defaults = struct( ... 'lineNoiseMethod', ... getRules('clean', {'char'}, {}, ... - 'Method for removing line noise (clean or blasst or none)'), ... + 'Method for removing line noise (clean or none)'), ... 'lineNoiseChannels', ... getRules(1:size(signal.data, 1), {'numeric'}, ... {'row', 'positive', 'integer', '<=', size(signal.data, 1)}, ... diff --git a/PrepPipeline/utilities/getPrepVersion.m b/PrepPipeline/utilities/getPrepVersion.m index 875cc12..213f08b 100644 --- a/PrepPipeline/utilities/getPrepVersion.m +++ b/PrepPipeline/utilities/getPrepVersion.m @@ -1,95 +1,95 @@ -function [currentVersion, changeLog, markdown] = getPrepVersion() - - changeLog = getChangeLog(); - currentVersion = ['PrepPipeline' changeLog(end).version]; - markdown = getMarkdown(changeLog); -end - -function changeLog = getChangeLog() - changeLog(7) = ... - struct('version', '0', 'status', 'Unreleased', 'date', '', 'changes', ''); - changeLog(7).version = '0.57.0'; - changeLog(7).status = 'Released'; - changeLog(7).date = '3/31/2025'; - changeLog(7).changes = {'Updated the interface for new EEGLAB'}; - - changeLog(6) = ... - struct('version', '0', 'status', 'Unreleased', 'date', '', 'changes', ''); - - changeLog(6).version = '0.56.0'; - changeLog(6).status = 'Released'; - changeLog(6).date = '8/01/2021'; - changeLog(6).changes = { ... - 'Corrected parfor failure when channel number not consecutive'; - 'Fixed missing badChannelsFromDropout in updateBadChannels issue#28'... - }; - - changeLog(5) = ... - struct('version', '0', 'status', 'Unreleased', 'date', '', 'changes', ''); - - changeLog(5).version = '0.55.4'; - changeLog(5).status = 'Released'; - changeLog(5).date = '7/26/2020'; - changeLog(5).changes = { ... - 'Correctly restored EEGLAB options after execution'; ... - 'Added functions to output errors from etc.noiseDetection'; ... - 'Corrected findpeaks naming conflict in Chronux'; ... - 'Post process does not execute if Prep had errors'}; - - changeLog(4) = ... - struct('version', '0', 'status', 'Unreleased', 'date', '', 'changes', ''); - - changeLog(4).version = '0.55.3'; - changeLog(4).status = 'Released'; - changeLog(4).date = '10/19/2017'; - changeLog(4).changes = { ... - 'Fixed issue with interpolated channels when interpolation order is pre-process'; ... - 'Fixed issue with correct removal of interpolated channels during post-processing'; ... - 'Reordered preprocessing and report buttons on master GUI'}; - - changeLog(3).version = '0.55.2'; - changeLog(3).status = 'Released'; - changeLog(3).date = '08/18/2017'; - changeLog(3).changes = { ... - 'Fixed undefined reference to referenceOut in prepPipeline post process'}; - - changeLog(2).version = '0.55.1'; - changeLog(2).status = 'Released'; - changeLog(2).date = '06/03/2017'; - changeLog(2).changes = { ... - 'Wrote printListCompressed to display channels more compactly'; ... - 'Put in a MATLAB version check because legend titles not supported in 2014b'; ... - 'Fixed spacing on output of interpolated channel numbers'}; - - changeLog(1).version = '0.55.0'; - changeLog(1).status = 'Released'; - changeLog(1).date = '05/29/2017'; - changeLog(1).changes = { ... - ['Changed the EEG.etc.noiseDetection structure to contain ' ... - 'removed channels and interpolated channels for easier access ']; ... - 'Fixed reporting to work when bad channels have been removed'; ... - ['Added original channel labels to EEG.etc.noiseDetection for ' ... - 'ease in reporting']; ... - 'Added Blasst as an unsupported line noise removal option'; ... - 'Moved legend of spectrum to right, put in checks for removed channels'; ... - 'Corrected bug in smoothing in cleanline'; ... - 'Corrected several reporting issues'; - 'Default behavior now outputs errors to command line in addition to logging'; ... - 'Renamed several functions to make naming scheme consistent'; ... - 'Started supporting changelog in versions'; ... - 'Fixed bug in struct2str and improved com return on pop_prepPipeline'}; -end - -function markdown = getMarkdown(changeLog) - markdown = ''; - for k = length(changeLog):-1:1 - tString = sprintf('Version %s %s %s\n', changeLog(k).version, ... - changeLog(k).status, changeLog(k).date); - changes = changeLog(k).changes; - for j = 1:length(changes) - cString = sprintf('* %s\n', changes{j}); - tString = [tString cString]; %#ok<*AGROW> - end - markdown = [markdown tString sprintf(' \n')]; - end +function [currentVersion, changeLog, markdown] = getPrepVersion() + + changeLog = getChangeLog(); + currentVersion = ['PrepPipeline' changeLog(end).version]; + markdown = getMarkdown(changeLog); +end + +function changeLog = getChangeLog() + changeLog(7) = ... + struct('version', '0', 'status', 'Unreleased', 'date', '', 'changes', ''); + changeLog(7).version = '0.57.0'; + changeLog(7).status = 'Released'; + changeLog(7).date = '3/31/2025'; + changeLog(7).changes = {'Updated the interface for new EEGLAB'}; + + changeLog(6) = ... + struct('version', '0', 'status', 'Unreleased', 'date', '', 'changes', ''); + + changeLog(6).version = '0.56.0'; + changeLog(6).status = 'Released'; + changeLog(6).date = '8/01/2021'; + changeLog(6).changes = { ... + 'Corrected parfor failure when channel number not consecutive'; + 'Fixed missing badChannelsFromDropout in updateBadChannels issue#28'... + }; + + changeLog(5) = ... + struct('version', '0', 'status', 'Unreleased', 'date', '', 'changes', ''); + + changeLog(5).version = '0.55.4'; + changeLog(5).status = 'Released'; + changeLog(5).date = '7/26/2020'; + changeLog(5).changes = { ... + 'Correctly restored EEGLAB options after execution'; ... + 'Added functions to output errors from etc.noiseDetection'; ... + 'Corrected findpeaks naming conflict in Chronux'; ... + 'Post process does not execute if Prep had errors'}; + + changeLog(4) = ... + struct('version', '0', 'status', 'Unreleased', 'date', '', 'changes', ''); + + changeLog(4).version = '0.55.3'; + changeLog(4).status = 'Released'; + changeLog(4).date = '10/19/2017'; + changeLog(4).changes = { ... + 'Fixed issue with interpolated channels when interpolation order is pre-process'; ... + 'Fixed issue with correct removal of interpolated channels during post-processing'; ... + 'Reordered preprocessing and report buttons on master GUI'}; + + changeLog(3).version = '0.55.2'; + changeLog(3).status = 'Released'; + changeLog(3).date = '08/18/2017'; + changeLog(3).changes = { ... + 'Fixed undefined reference to referenceOut in prepPipeline post process'}; + + changeLog(2).version = '0.55.1'; + changeLog(2).status = 'Released'; + changeLog(2).date = '06/03/2017'; + changeLog(2).changes = { ... + 'Wrote printListCompressed to display channels more compactly'; ... + 'Put in a MATLAB version check because legend titles not supported in 2014b'; ... + 'Fixed spacing on output of interpolated channel numbers'}; + + changeLog(1).version = '0.55.0'; + changeLog(1).status = 'Released'; + changeLog(1).date = '05/29/2017'; + changeLog(1).changes = { ... + ['Changed the EEG.etc.noiseDetection structure to contain ' ... + 'removed channels and interpolated channels for easier access ']; ... + 'Fixed reporting to work when bad channels have been removed'; ... + ['Added original channel labels to EEG.etc.noiseDetection for ' ... + 'ease in reporting']; ... + 'Added Blasst as an unsupported line noise removal option'; ... + 'Moved legend of spectrum to right, put in checks for removed channels'; ... + 'Corrected bug in smoothing in cleanline'; ... + 'Corrected several reporting issues'; + 'Default behavior now outputs errors to command line in addition to logging'; ... + 'Renamed several functions to make naming scheme consistent'; ... + 'Started supporting changelog in versions'; ... + 'Fixed bug in struct2str and improved com return on pop_prepPipeline'}; +end + +function markdown = getMarkdown(changeLog) + markdown = ''; + for k = length(changeLog):-1:1 + tString = sprintf('Version %s %s %s\n', changeLog(k).version, ... + changeLog(k).status, changeLog(k).date); + changes = changeLog(k).changes; + for j = 1:length(changes) + cString = sprintf('* %s\n', changes{j}); + tString = [tString cString]; %#ok<*AGROW> + end + markdown = [markdown tString sprintf(' \n')]; + end end \ No newline at end of file diff --git a/PrepPipeline/utilities/removeLineNoise.m b/PrepPipeline/utilities/removeLineNoise.m index 3c7fe1e..7f1c47f 100644 --- a/PrepPipeline/utilities/removeLineNoise.m +++ b/PrepPipeline/utilities/removeLineNoise.m @@ -65,8 +65,6 @@ if strcmpi(lineNoiseOut.lineNoiseMethod, 'clean') [signal, lineNoiseOut] = cleanLineNoise(signal, lineNoiseOut); -elseif strcmpi(lineNoiseOut.lineNoiseMethod, 'blasst') - [signal, lineNoiseOut] = blasstLineNoise(signal, lineNoiseOut); elseif ~strcmpi(lineNoiseOut.lineNoiseMethod, 'none') error('removeLineNoise:BadLineNoiseMethod', ... 'Unrecognized line noise removal method'); diff --git a/PrepPipeline/utilities/robustReference.m b/PrepPipeline/utilities/robustReference.m index 2a40bb0..5693b61 100644 --- a/PrepPipeline/utilities/robustReference.m +++ b/PrepPipeline/utilities/robustReference.m @@ -1,90 +1,90 @@ -function referenceOut = robustReference(signal, referenceOut) -% Robustly estimate of bad channels by iteratively interpolating channels -% -% This function finds bad channels by iteratively interpolating the -% bad list so far and calculating a mean of good signals. It assumes -% that defaults have already been checked on referenceIn. -% -% Parameters (input): -% signal structure with data field (assumes unfiltered) -% referenceOut structure with reference parameters with reference -% parameters in it. -% -% Parameters (output): -% referenceOut the referenceOut structure filled in - - -%% Warn if evaluation and reference channels are not the same for robust -if ~isempty( ... - setdiff(referenceOut.evaluationChannels,referenceOut.referenceChannels)) ... - || ~isempty( ... - setdiff(referenceOut.referenceChannels, referenceOut.evaluationChannels)) - warning('robustReference:EvaluationChannels', ... - 'Reference and evaluation channels should be same for robust reference'); -end - -%% Determine unusable channels and remove them from the reference channels -signal = removeTrend(signal, referenceOut); -referenceOut.noisyStatisticsOriginal = findNoisyChannels(signal, referenceOut); -referenceOut.noisyStatistics = referenceOut.noisyStatisticsOriginal; -[badChannelsFromNaNs, badChannelsFromNoData] = ... - findUnusableChannels(signal, referenceOut.referenceChannels); -noisy = referenceOut.noisyStatisticsOriginal.noisyChannels; -badChannelsFromLowSNR = noisy.badChannelsFromLowSNR; -unusableChannels = union(badChannelsFromNaNs, ... - union(badChannelsFromNoData, badChannelsFromLowSNR)); -unusableChannels = unusableChannels(:)'; -referenceOut.badChannels.badChannelsFromNaNs = ... - badChannelsFromNaNs(:)'; -referenceOut.badChannels.badChannelsFromNoData = ... - badChannelsFromNoData(:)'; -referenceOut.badChannels.badChannelsFromLowSNR = ... - badChannelsFromLowSNR(:)'; -referenceChannels = setdiff(referenceOut.referenceChannels, unusableChannels); - -%% Get initial estimate of the mean by the specified method -if strcmpi(referenceOut.meanEstimateType, 'median') - refTemp = median(signal.data(referenceChannels, :), 1); - signalTmp = removeReference(signal, refTemp, referenceChannels); -elseif strcmpi(referenceOut.meanEstimateType, 'mean') - refTemp = mean(signal.data(referenceChannels, :), 1); - signalTmp = removeReference(signal, refTemp, referenceChannels); -elseif strcmpi(referenceOut.meanEstimateType, 'huber') - signalTmp = removeHuberMean(signal, referenceChannels); -else - signalTmp = signal; -end - -%% Remove reference from signal iteratively interpolating bad channels -iterations = 0; -noisyChannelsOld = []; -while true % Do at least 1 iteration - noisyStatistics = findNoisyChannels(signalTmp, referenceOut); - referenceOut.badChannels = ... - updateBadChannels(referenceOut.badChannels, noisyStatistics.noisyChannels); - noisyChannels = referenceOut.badChannels.all(:)'; - if (iterations > 1 && (isempty(noisyChannels) ||... - (isempty(setdiff(noisyChannels, noisyChannelsOld)) ... - && isempty(setdiff(noisyChannelsOld, noisyChannels))))) || ... - iterations > referenceOut.maxReferenceIterations - break; - end - noisyChannelsOld = noisyChannels; - sourceChannels = setdiff(referenceOut.referenceChannels, noisyChannels); - if length(sourceChannels) < 2 - error('robustReference:TooManyBad', ... - 'Could not perform a robust reference -- not enough good channels'); - end - if ~isempty(noisyChannels) - signalTmp = interpolateChannels(signal, noisyChannels, sourceChannels); - else - signalTmp = signal; - end - referenceSignal = nanmean(signalTmp.data(referenceChannels, :), 1); - signalTmp = removeReference(signal, referenceSignal, referenceChannels); - iterations = iterations + 1; - fprintf('Iteration: %d\n', iterations); -end -referenceOut.actualReferenceIterations = iterations; -referenceOut.noisyStatistics = noisyStatistics; -fprintf('Robust reference done\n'); +function referenceOut = robustReference(signal, referenceOut) +% Robustly estimate of bad channels by iteratively interpolating channels +% +% This function finds bad channels by iteratively interpolating the +% bad list so far and calculating a mean of good signals. It assumes +% that defaults have already been checked on referenceIn. +% +% Parameters (input): +% signal structure with data field (assumes unfiltered) +% referenceOut structure with reference parameters with reference +% parameters in it. +% +% Parameters (output): +% referenceOut the referenceOut structure filled in + + +%% Warn if evaluation and reference channels are not the same for robust +if ~isempty( ... + setdiff(referenceOut.evaluationChannels,referenceOut.referenceChannels)) ... + || ~isempty( ... + setdiff(referenceOut.referenceChannels, referenceOut.evaluationChannels)) + warning('robustReference:EvaluationChannels', ... + 'Reference and evaluation channels should be same for robust reference'); +end + +%% Determine unusable channels and remove them from the reference channels +signal = removeTrend(signal, referenceOut); +referenceOut.noisyStatisticsOriginal = findNoisyChannels(signal, referenceOut); +referenceOut.noisyStatistics = referenceOut.noisyStatisticsOriginal; +[badChannelsFromNaNs, badChannelsFromNoData] = ... + findUnusableChannels(signal, referenceOut.referenceChannels); +noisy = referenceOut.noisyStatisticsOriginal.noisyChannels; +badChannelsFromLowSNR = noisy.badChannelsFromLowSNR; +unusableChannels = union(badChannelsFromNaNs, ... + union(badChannelsFromNoData, badChannelsFromLowSNR)); +unusableChannels = unusableChannels(:)'; +referenceOut.badChannels.badChannelsFromNaNs = ... + badChannelsFromNaNs(:)'; +referenceOut.badChannels.badChannelsFromNoData = ... + badChannelsFromNoData(:)'; +referenceOut.badChannels.badChannelsFromLowSNR = ... + badChannelsFromLowSNR(:)'; +referenceChannels = setdiff(referenceOut.referenceChannels, unusableChannels); + +%% Get initial estimate of the mean by the specified method +if strcmpi(referenceOut.meanEstimateType, 'median') + refTemp = median(signal.data(referenceChannels, :), 1); + signalTmp = removeReference(signal, refTemp, referenceChannels); +elseif strcmpi(referenceOut.meanEstimateType, 'mean') + refTemp = mean(signal.data(referenceChannels, :), 1); + signalTmp = removeReference(signal, refTemp, referenceChannels); +elseif strcmpi(referenceOut.meanEstimateType, 'huber') + signalTmp = removeHuberMean(signal, referenceChannels); +else + signalTmp = signal; +end + +%% Remove reference from signal iteratively interpolating bad channels +iterations = 0; +noisyChannelsOld = []; +while true % Do at least 1 iteration + noisyStatistics = findNoisyChannels(signalTmp, referenceOut); + referenceOut.badChannels = ... + updateBadChannels(referenceOut.badChannels, noisyStatistics.noisyChannels); + noisyChannels = referenceOut.badChannels.all(:)'; + if (iterations > 1 && (isempty(noisyChannels) ||... + (isempty(setdiff(noisyChannels, noisyChannelsOld)) ... + && isempty(setdiff(noisyChannelsOld, noisyChannels))))) || ... + iterations >= referenceOut.maxReferenceIterations + break; + end + noisyChannelsOld = noisyChannels; + sourceChannels = setdiff(referenceOut.referenceChannels, noisyChannels); + if length(sourceChannels) < 2 + error('robustReference:TooManyBad', ... + 'Could not perform a robust reference -- not enough good channels'); + end + if ~isempty(noisyChannels) + signalTmp = interpolateChannels(signal, noisyChannels, sourceChannels); + else + signalTmp = signal; + end + referenceSignal = nanmean(signalTmp.data(referenceChannels, :), 1); + signalTmp = removeReference(signal, referenceSignal, referenceChannels); + iterations = iterations + 1; + fprintf('Iteration: %d\n', iterations); +end +referenceOut.actualReferenceIterations = iterations; +referenceOut.noisyStatistics = noisyStatistics; +fprintf('Robust reference done\n'); diff --git a/PrepPipeline/utilities/struct2str.m b/PrepPipeline/utilities/struct2str.m index 21afed6..001bce7 100644 --- a/PrepPipeline/utilities/struct2str.m +++ b/PrepPipeline/utilities/struct2str.m @@ -1,47 +1,47 @@ -function [str] = struct2str(theStruct) -% Converts a struct into a string -str = ''; -fNames = fieldnames(theStruct); -if isempty(fNames) - return; -end -str = 'struct('; -for a = 1:length(fNames) - if ischar(theStruct.(fNames{a})) - strVal = getStr(a); - elseif islogical(theStruct.(fNames{a})) - strVal = getLogical(a); - else - strVal = getNumerical(a); - end - str = [str '''' fNames{a} ''', ' strVal]; %#ok -end -if strcmpi(str(end-1), ',') - str = str(1:end-2); -end -str = [str ')']; - - function strVal = getLogical(indx) - % Appends a logical structure field to the string - if theStruct.(fNames{indx}) - strVal = 'true, '; - else - strVal = 'false, '; - end - end % getLogical - - function strVal = getNumerical(indx) - % Appends a numerical structure field to the string - if isscalar(theStruct.(fNames{indx})) - strVal = [num2str(theStruct.(fNames{indx})) ', ']; - else - strVal = ['[' num2str(theStruct.(fNames{indx})) '], ']; - end - end % getNumerical - - function strVal = getStr(indx) - % Appends a string structure field to the string - strVal = ['''' theStruct.(fNames{indx}) ''', ']; - end % handleStr - +function [str] = struct2str(theStruct) +% Converts a struct into a string +str = ''; +fNames = fieldnames(theStruct); +if isempty(fNames) + return; +end +str = 'struct('; +for a = 1:length(fNames) + if ischar(theStruct.(fNames{a})) + strVal = getStr(a); + elseif islogical(theStruct.(fNames{a})) + strVal = getLogical(a); + else + strVal = getNumerical(a); + end + str = [str '''' fNames{a} ''', ' strVal]; %#ok +end +if strcmpi(str(end-1), ',') + str = str(1:end-2); +end +str = [str ')']; + + function strVal = getLogical(indx) + % Appends a logical structure field to the string + if theStruct.(fNames{indx}) + strVal = 'true, '; + else + strVal = 'false, '; + end + end % getLogical + + function strVal = getNumerical(indx) + % Appends a numerical structure field to the string + if isscalar(theStruct.(fNames{indx})) + strVal = [num2str(theStruct.(fNames{indx})) ', ']; + else + strVal = ['[' num2str(theStruct.(fNames{indx})) '], ']; + end + end % getNumerical + + function strVal = getStr(indx) + % Appends a string structure field to the string + strVal = ['''' theStruct.(fNames{indx}) ''', ']; + end % handleStr + end % struct2str \ No newline at end of file diff --git a/PrepPipeline/utilities/updateBadChannels.m b/PrepPipeline/utilities/updateBadChannels.m index 98abf6e..d79b947 100644 --- a/PrepPipeline/utilities/updateBadChannels.m +++ b/PrepPipeline/utilities/updateBadChannels.m @@ -1,30 +1,30 @@ -function ref = updateBadChannels(ref, noisy) -% Update the bad channel lists from ref based on bad channels in noisy - ref.badChannelsFromNaNs = union(ref.badChannelsFromNaNs, ... - noisy.badChannelsFromNaNs); - ref.badChannelsFromNoData = union(ref.badChannelsFromNoData, ... - noisy.badChannelsFromNoData); - ref.badChannelsFromHFNoise = union(ref.badChannelsFromHFNoise, ... - noisy.badChannelsFromHFNoise); - ref.badChannelsFromCorrelation = union(ref.badChannelsFromCorrelation, ... - noisy.badChannelsFromCorrelation); - ref.badChannelsFromDeviation = union(ref.badChannelsFromDeviation, ... - noisy.badChannelsFromDeviation); - ref.badChannelsFromRansac = union(ref.badChannelsFromRansac, ... - noisy.badChannelsFromRansac); - ref.badChannelsFromDropOuts = union(ref.badChannelsFromDropOuts, ... - noisy.badChannelsFromDropOuts); - ref.all = union(... - union(ref.badChannelsFromNaNs, ... - ref.badChannelsFromNoData), ... - union( ... - union(ref.badChannelsFromHFNoise, ... - ref.badChannelsFromCorrelation), ... - union( ... - union(ref.badChannelsFromDeviation, ... - ref.badChannelsFromRansac), ... - ref.badChannelsFromDropOuts ... - ) ... - ) ... - ); +function ref = updateBadChannels(ref, noisy) +% Update the bad channel lists from ref based on bad channels in noisy + ref.badChannelsFromNaNs = union(ref.badChannelsFromNaNs, ... + noisy.badChannelsFromNaNs); + ref.badChannelsFromNoData = union(ref.badChannelsFromNoData, ... + noisy.badChannelsFromNoData); + ref.badChannelsFromHFNoise = union(ref.badChannelsFromHFNoise, ... + noisy.badChannelsFromHFNoise); + ref.badChannelsFromCorrelation = union(ref.badChannelsFromCorrelation, ... + noisy.badChannelsFromCorrelation); + ref.badChannelsFromDeviation = union(ref.badChannelsFromDeviation, ... + noisy.badChannelsFromDeviation); + ref.badChannelsFromRansac = union(ref.badChannelsFromRansac, ... + noisy.badChannelsFromRansac); + ref.badChannelsFromDropOuts = union(ref.badChannelsFromDropOuts, ... + noisy.badChannelsFromDropOuts); + ref.all = union(... + union(ref.badChannelsFromNaNs, ... + ref.badChannelsFromNoData), ... + union( ... + union(ref.badChannelsFromHFNoise, ... + ref.badChannelsFromCorrelation), ... + union( ... + union(ref.badChannelsFromDeviation, ... + ref.badChannelsFromRansac), ... + ref.badChannelsFromDropOuts ... + ) ... + ) ... + ); \ No newline at end of file diff --git a/README.md b/README.md index 8a85df4..f8a73dc 100644 --- a/README.md +++ b/README.md @@ -1,231 +1,120 @@ -EEG-Clean-Tools -=============== - -Contains tools for the PREP pipeline for standardized preprocessing of EEG. You can -find user documention at: - http://vislab.github.io/EEG-Clean-Tools/ - -**Note:** For convenience, EEGLABPlugin directory contains the latest released version of the -PREP that can be unzipped into your EEGLAB plugins directory. - -### Citing the PREP pipeline -The PREP pipeline is freely available under the GNU General Public License. -Please cite the following publication if using: -> Bigdely-Shamlo N, Mullen T, Kothe C, Su K-M and Robbins KA (2015) -> The PREP pipeline: standardized preprocessing for large-scale EEG analysis -> Front. Neuroinform. 9:16. doi: 10.3389/fninf.2015.00016 - -### People -The PREP pipeline incorporates many algorithms that were developed at -USCS SCCN over many years by Nima Bigdely-Shamlo, Tim Mullen and Christian Kothe. -Kyung Min Su performed most of the machine learning evaluation of PREP. Cassidy -Matousek and Jeremy Cockfield worked on the interfaces for the EEGLAB plugin as -well as associated visualization tools. Kay Robbins of UTSA is the lead developer and -maintainer of PREP. - -### Support: -This research was sponsored by the Army Research Laboratory and was accomplished -under Cooperative Agreement Number W911NF-10-2-0022. The views and conclusions -contained in this document/software are those of the authors and should not be interpreted -as representing the official policies, either expressed or implied, of the -Army Research Laboratory or the U.S. Government. The U.S. Government is -authorized to reproduce and distribute reprints for Government purposes -notwithstanding any copyright notation herein. - -### Releases -Version 0.57.0 Released 3/30/2025 -* Modified to work with modified EEGLAB GUI Builder -* Modified reporting to not clutter workspace - -Version 0.56.0 Released 8/01/2021 -* Corrected parfor failure when channel number not consecutive -* Fixed missing badChannelsFromDropout in updateBadChannels issue#28 - -Version 0.55.4 Released 7/26/2020 -* Correctly restored EEGLAB options after execution -* Added functions to output errors from etc.noiseDetection -* Corrected findpeaks naming conflict in Chronux -* Post process does not execute if Prep had errors - -Version 0.55.3 Released 10/19/2017 -* Fixed issue with interpolated channels when interpolation order is pre-process -* Fixed issue with correct removal of interpolated channels during post-processing -* Reordered preprocessing and report buttons on master GUI - -Version 0.55.2 Released 08/18/2017 -* Fixed undefined reference to referenceOut in prepPipeline post process - -Version 0.55.1 Released 06/03/2017 -* Wrote printListCompressed to display channels more compactly -* Put in a MATLAB version check because legend titles not supported in 2014b -* Fixed spacing on output of interpolated channel numbers - -Version 0.55.0 Released 05/29/2017 -* Changed the EEG.etc.noiseDetection structure to contain removed channels and interpolated channels for easier access -* Fixed reporting to work when bad channels have been removed -* Added original channel labels to EEG.etc.noiseDetection for ease in reporting -* Added Blasst as an unsupported line noise removal option -* Moved legend of spectrum to right, put in checks for removed channels -* Corrected bug in smoothing in cleanline -* Corrected several reporting issues -* Default behavior now outputs errors to command line in addition to logging -* Renamed several functions to make naming scheme consistent -* Started supporting changelog in versions -* Fixed bug in struct2str and improved com return on pop_prepPipeline - -Version 0.52 Released -* Modified code to handle EEG structures with empty EEG.error. -* Performed additional minor cleanup. - -Version 0.51 Not released -* Developing bad window visualization plugin for EEGLAB - -Version 0.50 Released -* Made several cleanup modifications to ready for release. - -Version 0.48 (Not released -- version 0.47 with EEGLAB integration) -* Integrated EEGLAB plugin -* Changed the default structure value field name from defaults.default to - default.value and propagated the change -* Changed default names of line noise and global trend to linenoise and - globaltrend -* Modified the resampling step to allow an option low pass filter to remove - downsampling artifacts just below Nyquist frequency. - - Version 0.47 (Not released -- version 0.46 with additional changes) -* Minor refactoring of performReference to avoid 1 extra filtering operation --- - should not reflect results. -* Also added average and specific referencing methods -- not tested as yet. - -Version 0.46 (Not released - version 0.45 with additional changes) -* Fixed remapping of bad evaluation channels into original channel numbers - (relevant when there are none EEG channels interspersed in the channel - locations. -* Passed detrend information in reference structure to allow detrending - with other than the defaults -* Corrected several channel mapping issues in the reporting. - -Version 0.45 (Not released - version 0.44 with additional changes) -* Refactored report to allow statistics to be gathered from noisy structures - -Version 0.44 (Not released - version 0.43 with additional changes) -* Corrected a minor issue with reporting -- difference between robust - and ordinary reference had axes reversed. -* Updated to run with plotting compatible with MATLAB 2014b -* Added box on to cummulative plots. - -Version 0.43 (Not released - version 0.42 with additional changes) -* Corrected a minor issue with reporting -- mean scalp correlation map for - beforeInterpolation was plotting the Original data rather than the - beforeInterpolation data. - -Version 0.42 (Not released - version 0.41 with additional changes) -* Added default line frequencies as multiples of 60 up to half nyquist. - -Version 0.41 (Not released - version 0.40 with additional changes) -* Replaced default method with channel forgetting and median initialization -* Converted EEG to double at the beginning of the pipeline -* Added a noisyStatisticsForInterpolation field to the reference reporting - structure. - -Version 0.40 (Not yet released - major change in strategy) -* Changed the name from StandardLevel2 to PrepPipeline -* Implemented the HP filter-free strategy -* Added a keepFiltered version -- if false (the default) the data in the - repository is not high pass filtered -* Added an option for removing global trend -* Incorporated the different reference schemes into a single performReference - -Version 0.28 (Not yet released) -* Changed the name of the noisyParameter structure in EEG.etc to - noiseDetection. This is a major change with corresponding change - in ESS. -* Added a specificReferenceChannels field to reference structure -* Changed the averageReference field name to referenceSignal in reference - structure -* Included a referenceType field in the reference structure (this - can be 'robust', 'average', or 'specific') -* Eliminated the don't interpolateHFChannels flag. -* Added routines to do specificReference (mastoid or average) -* Modified showSpectrum to return the spectra of all of the channels. -* Detrending at 0.2 Hz has replaced FIR filtering as default trend removal. - -Version 0.27 Released 1/7/2015 -* Correct version of bug fix in cleanLineNoise -- watch that single - precision conversion! - -Version 0.26 Released 1/7/2015 -* Release to fix bug in cleanLineNoise --- channels that are not - lineNoiseChannels were set to zero rather than being carried forward. - - Version 0.25 Released 1/5/2015 (major) -* Removed saving of temporary file after line noise removal -* Fixed report of relative reference -* Modified findNoisyChannels to exclude NaN and constant channels - from noisyChannel thresholding, but to designate them as bad channels -* Moved resampling step before high pass filter -* Assigned return values in a separate step -* Put error check in ShowSpectrum when invalid data is invalid -* Correct minor issues with PlotScalpMap -* Added extractReferenceStatistics -- which extracts summary statistics - for an entire archive. -* Added iterations on the remove robust reference -* Added a summary reporting scheme for spotting problematic datasets. - -Version 0.24 Released 12/7/2104 (major) -* Fixed channel selection bug in showSpectrum -* Added error handling for failures in standardLevel2Pipeline -* Added error reporting for failures -* Corrected time scale on visualization of difference between - robust and mean reference -* Added channel labels as well as numbers to spectrum visualization -* Fixed major bug in robustReference so that original signal is rereferenced -* Revised and expanded the reporting - -Version 0.23 Released 11/13/2014 - -* Removed the channel locations and channel information from noisyOut - because it is already in the reference structure at top level. -* Added reporting of average fraction of channels bad in windows. -* Added first version of hdf5support -- rewrites the noisyParameters - to an HDF5 file. - -Version 0.22 Released 11/9/2014 - -* Revised the method of computing the windowed channel deviations -* Added summary reporting functions -* Added a check to only perform ransac when sufficiently good channels - are available -* Added check to only perform ransac when channel locations are available -* Fixed the input parameter structure on findNoisyChannels -* Added the infrastructure for the summary of all datasets - -Version 0.21 Released 10/30/2104 - -* Removed any reference to chanlocs in highPassFilter -* Full integration with ESS Study Level 2 code -* Preliminary version of Standard Level 2 Report finalized (gives pdf) - -Version 0.20 Released 10/18/2014 - -* Converted standardLevel2Pipeline to a function -* Moved the computationTimes structure to standardLevel2Pipeline so that -it is returned. - -Version 0.19 Released 10/16/2014 - -* Refactored name is also included in the params structure. -* Renamed rereferencedChannels as channelsToBeReferenced to agree with ESS. - -Version 0.18 Released 10/15/2014 - -* Refactored so that all input to the pipeline is in a single params structure. -* Fixed the HF noise reporting windows and several minor bugs -* Added visualizations to show number of bad channels in each window - - - - - - - +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22983431.svg)](https://doi.org/10.5281/zenodo.22983431) + +EEG-Clean-Tools +=============== + +Contains tools for the PREP pipeline for standardized preprocessing of EEG. You can +find the user documentation at +[https://vislab.github.io/EEG-Clean-Tools/](https://vislab.github.io/EEG-Clean-Tools/). + +**Note:** For convenience, EEGLABPlugin directory contains the latest released version of the +PREP that can be unzipped into your EEGLAB plugins directory. + +### Building the documentation +The documentation source is in `docs/` (Sphinx, with MyST markdown). To build and +view it locally, set up a Python virtual environment once, from the repository root. +Python 3.10 or later is required. + +**Windows (PowerShell):** + +```powershell +python -m venv --clear .venv +.venv\Scripts\Activate.ps1 +python -m pip install -e ".[docs]" +python docs/patch_matlabdomain.py +``` + +If PowerShell refuses to run `Activate.ps1`, allow local scripts for your account +once with `Set-ExecutionPolicy -Scope CurrentUser RemoteSigned`. In `cmd.exe`, activate +with `.venv\Scripts\activate.bat` instead. + +**Linux and macOS:** + +```bash +python3 -m venv --clear .venv +source .venv/bin/activate +python -m pip install -e ".[docs]" +python docs/patch_matlabdomain.py +``` + +On Debian and Ubuntu, `python3 -m venv` needs the `python3-venv` package +(`sudo apt install python3-venv`). + +`docs/patch_matlabdomain.py` must be rerun after every reinstall of +`sphinxcontrib-matlabdomain`. In later sessions, only the activation line is needed. + +With the environment activated, build and serve the site. These commands are the +same on every platform: + +```shell +python -m sphinx -b html docs docs/_build/html +python -m http.server 8000 --bind 127.0.0.1 -d docs/_build/html +``` + +The server prints `Serving HTTP on 127.0.0.1 port 8000 (http://127.0.0.1:8000/)`. +Open that address in a browser, and stop the server with Ctrl+C. The +`--bind 127.0.0.1` keeps the server reachable only from your own machine. Opening +`docs/_build/html/index.html` directly also works, but search does not. + +### Publishing the documentation +The site at https://vislab.github.io/EEG-Clean-Tools/ is served by GitHub Pages. +Pushing any branch other than `master` publishes nothing. The workflow +`.github/workflows/deploy-docs.yaml` builds the docs on pull requests to `master` +without deploying, and builds and deploys them on pushes to `master`. + +That deploy only reaches the site when the repository's Pages source (Settings -> +Pages -> Source) is "GitHub Actions". While the source is "Deploy from a branch: +gh-pages", the live site is the one on the `gh-pages` branch, and the workflow +cannot replace it. Do not switch the source, or delete `gh-pages`, until the +contents of `docs/` are ready to go live: the first Actions deployment overwrites +the site at the same address. + +### Citing the PREP pipeline +The PREP pipeline is freely available under the GNU General Public License (see License below). +Please cite the following publication if using: + +> Bigdely-Shamlo N, Mullen T, Kothe C, Su K-M and Robbins KA (2015)\ +> The PREP pipeline: standardized preprocessing for large-scale EEG analysis\ +> Front. Neuroinform. 9:16. doi: 10.3389/fninf.2015.00016 + +### License +The PREP pipeline is licensed under the GNU General Public License, version 2 or +(at your option) any later version. The full text is in [LICENSE](LICENSE), and a +copy is kept with the plugin as `PrepPipeline/preplicense.txt`. Parts of the +repository come from other projects and keep their own licenses: + +| Component | Location | License | Copyright | +| --- | --- | --- | --- | +| PREP pipeline | everything not listed below | GPL-2.0-or-later ([LICENSE](LICENSE)) | Kay Robbins, with contributions from Nima Bigdely-Shamlo, Christian Kothe, Tim Mullen, Jeremy Cockfield, and Cassidy Matousek | +| Chronux 2, modified | `PrepPipeline/utilities/chronux_2_modified/` | GPL-2.0 (`License.txt` in that folder) | The Chronux developers ([chronux.org](http://www.chronux.org/)) | +| Line-noise removal and local detrending, adapted from cleanline and Chronux | `PrepPipeline/utilities/cleanLineNoise.m` and the functions it calls (`removeLinesMovingWindow.m`, `fitSignificantFrequencies.m`, `calculateSegmentSpectrum.m`, `private/checkTapers.m`); `PrepPipeline/utilities/localDetrend.m` | GPL, as the code they adapt | cleanline by Tim Mullen, which builds on Chronux; adaptations by Kay Robbins | +| Spherical interpolation | `PrepPipeline/utilities/private/spherical_interpolate.m` | Permissive: use, copy, and modify, keeping the copyright notice and noting changes (file header) | Jason D.R. Farquhar; modified by Kay Robbins | +| Helpers from EEGLAB | `PrepPipeline/reporting/calculateSpectrum.m`, `reporting/helpers/finputcheck.m`, `reporting/helpers/matsel.m` | GPL-2.0-or-later (file headers) | Scott Makeig, Arnaud Delorme, and Marissa Westerfield, SCCN, UCSD | +| Filter helpers | `PrepPipeline/utilities/private/design_fir.m`, `filter_fast.m`, `filtfilt_fast.m`, `hlp_microcache.m` | GPL-2.0-or-later (file headers) | Christian Kothe, SCCN, UCSD; `filter_fast.m` includes `fftfilt.m` from Octave by John W. Eaton | +| Documentation styling and build helper | `docs/_static/custom.css`, `docs/_static/gh_icon_fix.js`, `docs/patch_matlabdomain.py`, and parts of `docs/conf.py` | MIT ([docs/license_hed_matlab.txt](docs/license_hed_matlab.txt)) | HED Standard Working Group (from hed-matlab) | +| Example EEG data | `PrepPipeline/examples/data/` | Creative Commons Attribution 4.0 International ([CC BY 4.0](https://creativecommons.org/licenses/by/4.0/)) | U.S. Army Research Laboratory and the authors of the dataset; cite Robbins, Su, and Hairston, "An 18-subject EEG data collection using a visual-oddball task, designed for benchmarking algorithms and headset performance comparisons", *Data in Brief* ([article](https://www.sciencedirect.com/science/article/pii/S2352340917306285), [full data on NITRC](https://www.nitrc.org/projects/vep_eeg_raw/)) | +| Released plugin | `EEGLABPlugin/PrepPipeline.zip` | As its contents, above | Each zip is a snapshot of `PrepPipeline/` at its release | + +The PREP pipeline is designed and distributed for research purposes only and +should not be used for medical purposes. The authors accept no responsibility +for its use in this manner. + +### People +The PREP pipeline incorporates many algorithms that were developed at +USCS SCCN over many years by Nima Bigdely-Shamlo, Tim Mullen and Christian Kothe. +Kyung Min Su performed most of the machine learning evaluation of PREP. Cassidy +Matousek and Jeremy Cockfield worked on the interfaces for the EEGLAB plugin as +well as associated visualization tools. Kay Robbins of UTSA is the lead developer and +maintainer of PREP. + +### Support: +This research was sponsored by the Army Research Laboratory and was accomplished +under Cooperative Agreement Number W911NF-10-2-0022. The views and conclusions +contained in this document/software are those of the authors and should not be interpreted +as representing the official policies, either expressed or implied, of the +Army Research Laboratory or the U.S. Government. The U.S. Government is +authorized to reproduce and distribute reprints for Government purposes +notwithstanding any copyright notation herein. + +### Releases +Release history: [CHANGELOG.md](CHANGELOG.md). diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 0000000..2501b20 --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,267 @@ +/* Custom styles for the PREP pipeline docs - Furo theme */ + +/* Project name styling below logo */ +.sidebar-brand-text { + font-size: 1.5rem !important; + font-weight: 600 !important; + color: #0969da !important; + margin-top: 0.5rem !important; + text-align: center !important; +} + +html[data-theme="dark"] .sidebar-brand-text, +body[data-theme="dark"] .sidebar-brand-text { + color: #58a6ff !important; +} + +/* Make all heading levels match sidebar title blue color */ +.content h1, +.content h2, +.content h3, +.content h4, +.content h5, +.content h6, +article h1, +article h2, +article h3, +article h4, +article h5, +article h6 { + color: #0969da !important; +} + +body[data-theme="dark"] .content h1, +body[data-theme="dark"] .content h2, +body[data-theme="dark"] .content h3, +body[data-theme="dark"] .content h4, +body[data-theme="dark"] .content h5, +body[data-theme="dark"] .content h6, +body[data-theme="dark"] article h1, +body[data-theme="dark"] article h2, +body[data-theme="dark"] article h3, +body[data-theme="dark"] article h4, +body[data-theme="dark"] article h5, +body[data-theme="dark"] article h6 { + color: #58a6ff !important; +} + +@media (prefers-color-scheme: dark) { + body:not([data-theme="light"]) .content h1, + body:not([data-theme="light"]) .content h2, + body:not([data-theme="light"]) .content h3, + body:not([data-theme="light"]) .content h4, + body:not([data-theme="light"]) .content h5, + body:not([data-theme="light"]) .content h6, + body:not([data-theme="light"]) article h1, + body:not([data-theme="light"]) article h2, + body:not([data-theme="light"]) article h3, + body:not([data-theme="light"]) article h4, + body:not([data-theme="light"]) article h5, + body:not([data-theme="light"]) article h6 { + color: #58a6ff !important; + } +} + +/* Quick Links sidebar styling - darker gray box with proper padding */ +.sidebar-quicklinks { + margin: 1rem 0.75rem 1rem 0.75rem; + padding: 0.75rem 1rem; + background-color: #e8e8e8; + border-radius: 0.25rem; + border: 1px solid #d0d0d0; +} + +/* Style RST sidebar directive boxes to match left sidebar styling */ +.sidebar, +aside.sidebar { + background-color: #e8e8e8 !important; + border: 1px solid #d0d0d0 !important; + border-radius: 0.25rem !important; + padding: 0.75rem 1rem !important; +} + +body[data-theme="dark"] .sidebar, +body[data-theme="dark"] aside.sidebar { + background-color: #0d0d0d !important; + border-color: #1a1a1a !important; +} + +@media (prefers-color-scheme: dark) { + body:not([data-theme="light"]) .sidebar, + body:not([data-theme="light"]) aside.sidebar { + background-color: #0d0d0d !important; + border-color: #1a1a1a !important; + } +} + +.sidebar p.sidebar-title, +aside.sidebar p.sidebar-title { + font-weight: 600 !important; + margin-bottom: 0.5rem !important; +} + +/* Dark mode styling for quick links - much darker background */ +body[data-theme="dark"] .sidebar-quicklinks { + background-color: #0d0d0d; + border-color: #1a1a1a; +} + +/* Auto mode with dark system preference */ +@media (prefers-color-scheme: dark) { + body:not([data-theme="light"]) .sidebar-quicklinks { + background-color: #0d0d0d; + border-color: #1a1a1a; + } +} + +.sidebar-quicklinks h3 { + margin-top: 0; + margin-bottom: 0.5rem; + color: var(--color-sidebar-link-text); + font-size: var(--sidebar-item-font-size); + font-weight: 600; +} + +.sidebar-quicklinks ul { + list-style: none; + padding-left: 0; + margin-bottom: 0; +} + +.sidebar-quicklinks li { + margin-bottom: 0.25rem; +} + +.sidebar-quicklinks a { + color: var(--color-link) !important; + text-decoration: none; + font-size: var(--sidebar-item-font-size); + display: block; + padding: 0.25rem 0; + transition: color 0.2s ease; +} + +.sidebar-quicklinks a:hover { + color: var(--color-link-hover) !important; + text-decoration: underline; +} + +/* External link icon for quick links */ +.sidebar-quicklinks a[target="_blank"]::after { + content: " \2197"; + font-size: 0.7em; + opacity: 0.6; +} + +/* Make search field more visible with same background as quick links */ +.sidebar-search-container { + margin: 0 0.75rem; +} + +.sidebar-search-container input { + background-color: #e8e8e8 !important; + border: 1px solid #d0d0d0 !important; + color: var(--color-foreground-primary) !important; + background-image: url("data:image/svg+xml;charset=utf-8,%3Csvg xmlns='http://www.w3.org/2000/svg' width='24' height='24' viewBox='0 0 24 24' fill='none' stroke='%23666666' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Ccircle cx='11' cy='11' r='8'%3E%3C/circle%3E%3Cline x1='21' y1='21' x2='16.65' y2='16.65'%3E%3C/line%3E%3C/svg%3E") !important; + background-repeat: no-repeat !important; + background-position: right 0.5rem center !important; + background-size: 1.2em !important; + padding-right: 2.5rem !important; +} + +body[data-theme="dark"] .sidebar-search-container input { + background-color: #0d0d0d !important; + border: 1px solid #1a1a1a !important; + color: #e8e8e8 !important; + background-image: url("data:image/svg+xml;charset=utf-8,%3Csvg xmlns='http://www.w3.org/2000/svg' width='24' height='24' viewBox='0 0 24 24' fill='none' stroke='%23cccccc' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Ccircle cx='11' cy='11' r='8'%3E%3C/circle%3E%3Cline x1='21' y1='21' x2='16.65' y2='16.65'%3E%3C/line%3E%3C/svg%3E") !important; +} + +/* Auto mode with dark system preference */ +@media (prefers-color-scheme: dark) { + body:not([data-theme="light"]) .sidebar-search-container input { + background-color: #0d0d0d !important; + border: 1px solid #1a1a1a !important; + color: #e8e8e8 !important; + background-image: url("data:image/svg+xml;charset=utf-8,%3Csvg xmlns='http://www.w3.org/2000/svg' width='24' height='24' viewBox='0 0 24 24' fill='none' stroke='%23cccccc' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Ccircle cx='11' cy='11' r='8'%3E%3C/circle%3E%3Cline x1='21' y1='21' x2='16.65' y2='16.65'%3E%3C/line%3E%3C/svg%3E") !important; + } +} + +/* Make search icon visible */ +.sidebar-search-container input::placeholder { + color: var(--color-foreground-muted) !important; + opacity: 1 !important; +} + +body[data-theme="dark"] .sidebar-search-container input::placeholder { + color: #999999 !important; +} + +/* Style search icon/button visibility */ +.sidebar-search .icon { + color: var(--color-foreground-muted) !important; +} + +body[data-theme="dark"] .sidebar-search .icon { + color: #999999 !important; +} + +/* Hide view source and edit buttons in top-right header */ +a.muted-link[href*="_sources"], +a.muted-link[title*="Edit this page"], +a.muted-link[title*="View page source"], +.content-icon-container a[href*="_sources"] { + display: none !important; +} + +/* Ensure GitHub repository button shows as icon in top-right header */ + +/* Target the hijacked link using the class added by JS, OR the href attribute as fallback */ +.content-icon-container a.github-repo-link, +.content-icon-container a[href*="github.com"]:not([href*="/edit/"]) { + display: inline-flex !important; + align-items: center !important; + justify-content: center !important; + width: 2rem !important; + height: 2rem !important; + + /* GitHub Icon */ + background-image: url("data:image/svg+xml,%3Csvg viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12' fill='%23666666'/%3E%3C/svg%3E") !important; + background-repeat: no-repeat !important; + background-position: center !important; + background-size: 1.2rem !important; + + color: transparent !important; /* Hide any text */ + overflow: hidden !important; +} + +/* Hide the original SVG icon inside the link */ +.content-icon-container a.github-repo-link svg, +.content-icon-container a[href*="github.com"] svg { + display: none !important; +} + +/* Dark mode icon color */ +body[data-theme="dark"] .content-icon-container a.github-repo-link, +body[data-theme="dark"] .content-icon-container a[href*="github.com"]:not([href*="/edit/"]) { + background-image: url("data:image/svg+xml,%3Csvg viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12' fill='%23cccccc'/%3E%3C/svg%3E") !important; +} + +/* Explicitly hide the edit link via CSS as well */ +.content-icon-container a[href*="/edit/"] { + display: none !important; +} + +/* Logo styling - ensure visibility in both light and dark modes */ +.sidebar-brand-text { + font-weight: 600; +} + +/* Adjust logo size if needed */ +.sidebar-logo { + margin-bottom: 0.5rem; +} + +.sidebar-logo img { + max-width: 100%; + height: auto; +} diff --git a/docs/_static/gh_icon_fix.js b/docs/_static/gh_icon_fix.js new file mode 100644 index 0000000..82ade25 --- /dev/null +++ b/docs/_static/gh_icon_fix.js @@ -0,0 +1,39 @@ +document.addEventListener("DOMContentLoaded", function() { + // Function to fix the icons + function fixGitHubIcons() { + // Furo puts icons in .content-icon-container + // We look for links that point to GitHub + const links = document.querySelectorAll(".content-icon-container a"); + + links.forEach(link => { + const href = link.getAttribute("href"); + if (!href) return; + + // Check if it's a GitHub link (edit or blob/view) + if (href.includes("github.com")) { + + // If it's the Edit link, hide it + if (href.includes("/edit/")) { + link.style.display = "none"; + link.classList.add("hidden-edit-link"); // Marker for CSS + } + // If it's the View/Blob link, hijack it + else if (href.includes("/blob/") || href.includes("/tree/")) { + // Change URL to repo root + link.href = "https://github.com/VisLab/EEG-Clean-Tools"; + link.title = "Go to repository"; + link.setAttribute("aria-label", "Go to repository"); + + // Remove any text content (like "View source") to ensure only icon shows + // But keep the SVG if we were using the original, but we are replacing it via CSS. + // Safest is to empty the text content but keep the element structure if needed. + // Actually, Furo puts an SVG inside. We want to hide that SVG and show our own background. + link.classList.add("github-repo-link"); // Add class for CSS targeting + link.style.display = "inline-flex"; + } + } + }); + } + + fixGitHubIcons(); +}); diff --git a/docs/_static/images/MATLABWorkspace.png b/docs/_static/images/MATLABWorkspace.png new file mode 100644 index 0000000..7109038 Binary files /dev/null and b/docs/_static/images/MATLABWorkspace.png differ diff --git a/docs/_static/images/ParallelProcessing.png b/docs/_static/images/ParallelProcessing.png new file mode 100644 index 0000000..4b4929e Binary files /dev/null and b/docs/_static/images/ParallelProcessing.png differ diff --git a/docs/_static/images/PrepBoundary.png b/docs/_static/images/PrepBoundary.png new file mode 100644 index 0000000..e682762 Binary files /dev/null and b/docs/_static/images/PrepBoundary.png differ diff --git a/docs/_static/images/PrepDetrend.png b/docs/_static/images/PrepDetrend.png new file mode 100644 index 0000000..7ded9b9 Binary files /dev/null and b/docs/_static/images/PrepDetrend.png differ diff --git a/docs/_static/images/PrepFinalSave.png b/docs/_static/images/PrepFinalSave.png new file mode 100644 index 0000000..b867652 Binary files /dev/null and b/docs/_static/images/PrepFinalSave.png differ diff --git a/docs/_static/images/PrepFromEEGLAB.png b/docs/_static/images/PrepFromEEGLAB.png new file mode 100644 index 0000000..710277b Binary files /dev/null and b/docs/_static/images/PrepFromEEGLAB.png differ diff --git a/docs/_static/images/PrepLineNoiseParameters.png b/docs/_static/images/PrepLineNoiseParameters.png new file mode 100644 index 0000000..f68770f Binary files /dev/null and b/docs/_static/images/PrepLineNoiseParameters.png differ diff --git a/docs/_static/images/PrepMainMenu.png b/docs/_static/images/PrepMainMenu.png new file mode 100644 index 0000000..e8e4e77 Binary files /dev/null and b/docs/_static/images/PrepMainMenu.png differ diff --git a/docs/_static/images/PrepPostProcess.png b/docs/_static/images/PrepPostProcess.png new file mode 100644 index 0000000..21ac84e Binary files /dev/null and b/docs/_static/images/PrepPostProcess.png differ diff --git a/docs/_static/images/PrepReferenceParameters.png b/docs/_static/images/PrepReferenceParameters.png new file mode 100644 index 0000000..5c8db8b Binary files /dev/null and b/docs/_static/images/PrepReferenceParameters.png differ diff --git a/docs/_static/images/PrepReportParameters.png b/docs/_static/images/PrepReportParameters.png new file mode 100644 index 0000000..ac7df5c Binary files /dev/null and b/docs/_static/images/PrepReportParameters.png differ diff --git a/docs/_static/images/PrepResampleParameters.png b/docs/_static/images/PrepResampleParameters.png new file mode 100644 index 0000000..71f3f14 Binary files /dev/null and b/docs/_static/images/PrepResampleParameters.png differ diff --git a/docs/_templates/quicklinks.html b/docs/_templates/quicklinks.html new file mode 100644 index 0000000..1be7ce9 --- /dev/null +++ b/docs/_templates/quicklinks.html @@ -0,0 +1,9 @@ +
+

Quick links

+ +
diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..60f80f9 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,57 @@ +API reference +============= + +The public entry points of the PREP pipeline. Each function's documentation is +taken from the help text in its ``.m`` file. + +Running the pipeline +-------------------- + +.. mat:currentmodule:: . + +.. mat:autofunction:: prepPipeline + +.. mat:autofunction:: pop_prepPipeline + +.. mat:autofunction:: prepPostProcess + +Pipeline steps +-------------- + +.. mat:autofunction:: utilities.removeTrend + +.. mat:autofunction:: utilities.removeLineNoise + +.. mat:autofunction:: utilities.cleanLineNoise + +.. mat:autofunction:: utilities.performReference + +.. mat:autofunction:: utilities.findNoisyChannels + +.. mat:autofunction:: utilities.robustReference + +.. mat:autofunction:: utilities.interpolateChannels + +Defaults and version +-------------------- + +.. mat:autofunction:: utilities.getPrepDefaults + +.. mat:autofunction:: utilities.outputPrepDefaults + +.. mat:autofunction:: reporting.showPrepDefaults + +.. mat:autofunction:: utilities.getPrepVersion + +Reporting +--------- + +.. mat:currentmodule:: . + +.. mat:autofunction:: prepReport + +.. mat:autofunction:: publishPrepReport + +.. mat:autofunction:: reporting.extractReferenceStatistics + +.. mat:autofunction:: reporting.createCollectionStatistics diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..2f85586 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,121 @@ +# Configuration file for the Sphinx documentation builder. +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +import os +import re +from datetime import datetime, timezone + +DOCS_DIR = os.path.dirname(os.path.abspath(__file__)) +PREP_DIR = os.path.abspath(os.path.join(DOCS_DIR, "..", "PrepPipeline")) + + +def _prep_version(): + """Return the newest version listed in PrepPipeline/utilities/getPrepVersion.m.""" + path = os.path.join(PREP_DIR, "utilities", "getPrepVersion.m") + with open(path, encoding="utf-8") as f: + entries = re.findall(r"changeLog\((\d+)\)\.version\s*=\s*'([^']+)'", f.read()) + return max(entries, key=lambda e: int(e[0]))[1] if entries else "unknown" + + +# -- Project information ----------------------------------------------------- + +project = "PREP pipeline" +copyright = f"2014-{datetime.now(timezone.utc).year}, Kay Robbins and the PREP contributors" +author = "Kay Robbins" +version = _prep_version() +release = version + +# -- General configuration --------------------------------------------------- + +extensions = [ + "myst_parser", + "sphinxcontrib.matlab", + "sphinx.ext.autodoc", + "sphinx_copybutton", +] + +# MATLAB source: the pipeline folder. Vendored third-party code is parsed but +# not documented; api.rst lists only PREP's own entry points. +matlab_src_dir = PREP_DIR +matlab_short_links = True +matlab_auto_link = True +matlab_keep_package_prefix = False + +primary_domain = "mat" +add_module_names = False +myst_heading_anchors = 4 +myst_enable_extensions = [ + "colon_fence", + "deflist", + "html_image", + "linkify", + "substitution", +] + +templates_path = ["_templates"] +source_suffix = {".rst": "restructuredtext", ".md": "markdown"} +master_doc = "index" +exclude_patterns = ["_build", "_templates", "Thumbs.db", ".DS_Store"] + +# -- Options for HTML output ------------------------------------------------- + +pygments_style = "sphinx" +pygments_dark_style = "monokai" + +html_theme = "furo" +html_title = f"PREP pipeline {version}" + +html_theme_options = { + "light_css_variables": { + "color-brand-primary": "#0969da", + "color-brand-content": "#0969da", + }, + "dark_css_variables": { + "color-brand-primary": "#58a6ff", + "color-brand-content": "#58a6ff", + }, + "source_repository": "https://github.com/VisLab/EEG-Clean-Tools/", + "source_branch": "master", + "source_directory": "docs/", +} + +html_sidebars = { + "**": [ + "sidebar/brand.html", + "sidebar/search.html", + "sidebar/scroll-start.html", + "sidebar/navigation.html", + "quicklinks.html", + "sidebar/scroll-end.html", + ] +} + +html_static_path = ["_static"] +html_css_files = ["custom.css"] +html_js_files = ["gh_icon_fix.js"] + + +# -- MATLAB help text --------------------------------------------------------- + +# PREP's help text is written for MATLAB's `help` command: aligned columns and +# indented continuation lines, not reStructuredText. Parsed as reST it renders +# badly and produces docutils warnings, so show each docstring as a literal +# block - the same text, laid out exactly as `help` prints it. + + +_ROLE = re.compile(r":[a-z]+:`([^`]+)`") +_LITERAL = re.compile(r"``([^`]+)``") + + +def _help_text_as_literal(app, what, name, obj, options, lines): + if not any(line.strip() for line in lines): + return + # matlabdomain rewrites "See also" names into :func:`x` / ``x`` markup, + # which a literal block would show verbatim; restore the plain names. + plain = [_LITERAL.sub(r"\1", _ROLE.sub(r"\1", line)) for line in lines] + body = [(" " + line) if line.strip() else "" for line in plain] + lines[:] = ["::", "", *body, ""] + + +def setup(app): + app.connect("autodoc-process-docstring", _help_text_as_literal) diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..463ef4a --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,34 @@ +PREP pipeline +============= + +The PREP pipeline is a standardized early-stage EEG processing pipeline. It +identifies bad channels, removes line noise without committing to a filtering +strategy, and computes a robust average reference, with an extensive reporting +facility. It runs fully automated, as a MATLAB toolbox or as an EEGLAB plugin. + +If you use PREP, please cite: + + | Bigdely-Shamlo N, Mullen T, Kothe C, Su K-M and Robbins KA (2015). + | The PREP pipeline: standardized preprocessing for large-scale EEG analysis. + | *Front. Neuroinform.* 9:16. `doi:10.3389/fninf.2015.00016 `_ + +Getting started +--------------- + +.. toctree:: + :maxdepth: 2 + + User guide + +API documentation +----------------- + +.. toctree:: + :maxdepth: 2 + + API reference + +Index +----- + +* :ref:`genindex` diff --git a/docs/license_hed_matlab.txt b/docs/license_hed_matlab.txt new file mode 100644 index 0000000..39c6a36 --- /dev/null +++ b/docs/license_hed_matlab.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020- HED Standard Working Group + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/docs/patch_matlabdomain.py b/docs/patch_matlabdomain.py new file mode 100644 index 0000000..4961256 --- /dev/null +++ b/docs/patch_matlabdomain.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +""" +Patch sphinxcontrib-matlabdomain to fix compatibility issues with Sphinx 7.x+ + +This script fixes a bug in sphinxcontrib-matlabdomain where it uses +inspect.get_members() which doesn't exist in Python's standard library. +The correct function is inspect.getmembers(), but it has a different signature. +""" + +import importlib.util +import sys +from pathlib import Path + + +def find_matlabdomain_file(): + """Find the mat_documenters.py file in installed packages.""" + try: + # Use importlib to find the module spec + spec = importlib.util.find_spec("sphinxcontrib.mat_documenters") + + if spec and spec.origin: + return Path(spec.origin) + + # If that doesn't work, try to find the sphinxcontrib package directory + import sphinxcontrib + + # sphinxcontrib is often a namespace package, so __file__ might be None + if hasattr(sphinxcontrib, "__path__"): + # Iterate through the package paths + for pkg_path in sphinxcontrib.__path__: + pkg_path = Path(pkg_path) + + # Check for mat_documenters.py directly in sphinxcontrib + mat_doc_file = pkg_path / "mat_documenters.py" + if mat_doc_file.exists(): + return mat_doc_file + + # Check in nested matlab subdirectory + mat_doc_file = pkg_path / "matlab" / "mat_documenters.py" + if mat_doc_file.exists(): + return mat_doc_file + + print(f"Searched in: {pkg_path}") + + print("Could not locate mat_documenters.py") + return None + + except ImportError as e: + print(f"Error importing sphinxcontrib: {e}") + return None + except (AttributeError, FileNotFoundError, ModuleNotFoundError) as e: + print(f"Error finding sphinxcontrib.mat_documenters: {e}") + return None + + +def patch_file(file_path): + """Apply the patch to fix the inspect.get_members issue and MatScript args.""" + if not file_path: + print("mat_documenters.py not found - this may be expected for newer versions") + print("Skipping patch (not required)") + return True # Return success since patch may not be needed + + if not file_path.exists(): + print("Error: mat_documenters.py not found") + return True # Return success since file doesn't exist + + print(f"Checking {file_path}...") + + try: + content = file_path.read_text(encoding="utf-8") + patched = False + + # Patch 1: Fix inspect.get_members issue + if "for name in dir(self.object)" in content: + print("Already patched (inspect.get_members), skipping.") + else: + old_code = "members = inspect.get_members(self.object, attr_getter=self.get_attr)" + if old_code in content: + new_code = "members = [(name, self.get_attr(self.object, name)) for name in dir(self.object)]" + content = content.replace(old_code, new_code) + print("OK: Patch 1 applied: Fixed inspect.get_members") + patched = True + else: + print("OK: Bug not found in current version - patch 1 not needed") + + # Patch 2: Fix MatScript args formatting warning + if "hasattr(self.object, 'args')" not in content: + # Find the format_args method and add a check for MatScript + old_pattern = "def format_args(self, **kwargs):" + if old_pattern in content: + # Add check for scripts (which don't have args attribute) + lines = content.split("\n") + new_lines = [] + for i, line in enumerate(lines): + new_lines.append(line) + if old_pattern in line and i + 1 < len(lines): + # Find the indentation of the next line + next_line = lines[i + 1] + indent = len(next_line) - len(next_line.lstrip()) + # Insert check for args attribute (for MatScript objects) + new_lines.append(" " * indent + "if not hasattr(self.object, 'args'):") + new_lines.append(" " * indent + " return ''") + content = "\n".join(new_lines) + print("OK: Patch 2 applied: Fixed MatScript args warning") + patched = True + else: + print("OK: format_args method not found - patch 2 not needed") + else: + print("Already patched (MatScript args), skipping.") + + if patched: + file_path.write_text(content, encoding="utf-8") + print("OK: All patches applied successfully!") + else: + print("OK: No patches needed") + + return True + except OSError as e: + print(f"Error applying patch: {e}") + return False + + +if __name__ == "__main__": + file_path = find_matlabdomain_file() + success = patch_file(file_path) + sys.exit(0 if success else 1) diff --git a/docs/user_guide.md b/docs/user_guide.md new file mode 100644 index 0000000..328cec6 --- /dev/null +++ b/docs/user_guide.md @@ -0,0 +1,364 @@ +# PREP user guide + +## Introduction to the PREP pipeline +The PREP pipeline is a standardized early-stage EEG processing pipeline that focuses on the identification of bad channels and the calculation of a robust average reference. PREP also has an extensive reporting facility. It is designed to be run in a completely automated way. The major sections of this document are: +* Introduction (requirements, citing, installation) +* Algorithm (steps, meaning of parameters for each step) +* Running as an EEGLAB plug-in +* Running as a script + +### Requirements +The PREP pipeline relies on the MATLAB Signal Processing toolbox and EEGLAB, a freely-available MATLAB toolbox for processing EEG. EEGLAB is available from [https://sccn.ucsd.edu/eeglab](https://sccn.ucsd.edu/eeglab). PREP assumes that the EEG data is provided as an EEGLAB EEG structure and that channel locations are provided in the EEG.chanlocs structure. + +### Citing the PREP pipeline +The PREP pipeline is freely available under the GNU General Public License. +Please cite the following publication if using: +> Bigdely-Shamlo N, Mullen T, Kothe C, Su K-M and Robbins KA (2015)\ +> The PREP pipeline: standardized preprocessing for large-scale EEG analysis\ +> Front. Neuroinform. 9:16. doi: 10.3389/fninf.2015.00016 + +### Installation +The PREP pipeline can be run in two ways --- as a standalone toolbox or as an EEGLAB plugin. To run in standalone mode, simply download the EEG-Clean-Tools +repository from https://github.com/VisLab/EEG-Clean-Tools. Unzip if necessary and then add the PrepPipeline directory and all of its subdirectories to your +MATLAB path. + +## PREP as an EEGLAB plugin +You can install PREP as an EEGLAB plugin by unzipping `EEGLABPlugin/PrepPipeline.zip` from this +repository into the `plugins` directory of your EEGLAB installation. The zip holds a single +`PrepPipeline` folder; restart EEGLAB and PREP appears under the Tools menu. + +## Using parallel processing with PREP +The PREP pipeline can execute fairly slowly on headsets with a lot of channels. However, many of the steps are embarrassingly parallel --- that is the PREP can perform operations separately on individual channels or individual windows. +If you have the MATLAB Parallel Processing Toolbox, you just need to make sure that it is enabled. The following screenshot +of the MATLAB IDE shows the Parallel Processing Toolbox icon on the lower left of the status bar at the very bottom of the window. +![MATLAB IDE](_static/images/MATLABWorkspace.png) + +To configure your parallel processing toolbox, you should start the worker pool or set MATLAB to automatically start +the pool when needed: +![MATLAB Parallel processing](_static/images/ParallelProcessing.png) + +## Running the PREP pipeline from EEGLAB +Load an EEG dataset to be processed using the Load dataset submenu under the File menu of EEGLAB. The PREP pipeline +can be found under the EEGLAB Tools submenu: +![PREP from EEGLAB](_static/images/PrepFromEEGLAB.png) + +**[MAIN MENU]** The PREP main menu shows the processing steps: +![PREP main menu for EEGLAB plugin](_static/images/PrepMainMenu.png)\ +Each button allows you to override the default parameters. When you press +the Ok button (or unfortunately the Cancel button), PREP runs. Use the x button on the upper right to quit without +running the pipeline. + +**[BOUNDARY MENU]** Normally, PREP won't run on data sets that contain Boundary events. The PREP boundary menu allows you to change how PREP handles boundary events: +![PREP boundary menu](_static/images/PrepBoundary.png)\ +EEGLAB inserts Boundary events to mark discontinuities in the data from epoch rejection or when the data is imported. +Some, but not all, EEGLAB functions respect discontinuities. PREP expects the data set to be continuous. You should not +override this default setting unless the Boundary events in your data set do not mark discontinuous recording, but +other features. + +**[DETREND MENU]** PREP must detrend or high pass filter the data prior detecting line noise or referencing in order to properly calibrate the thresholds: +![PREP detrend menu](_static/images/PrepDetrend.png)\ +By default, PREP uses a high pass filter at 1 Hz for this purpose and does not retain the filtered version of the final +output. This allows you to defer the decision of filtering strategy to downstream processing. Normally, the only default you might need to over ride is what channels to filter. By default, PREP filters all of the channels. + +**[LINE NOISE MENU]** PREP tries to remove sharp spectral peaks representing line noise at this step: +![PREP line noise menu](_static/images/PrepLineNoiseParameters.png)\ +By default, PREP tries to remove multiples of 60 Hz up to the Nyquist frequency (half of the sampling frequency). +You may need to override this if your data set has unusual spectral features. You might also need to specify +which channels should have line noise removed. By default, PREP tries to remove line noise from all channels. + +**[REFERENCE MENU]** The PREP settings for the actual reference step are: +![PREP reference menu](_static/images/PrepReferenceParameters.png)\ +Normally, the only things you will need to override are the channel specifications. Usually, the reference channels and +the evaluation channels correspond to brain EEG channels. The re-referenced channels may include additional channels +such as EOG channels and mastoids. + +**[REPORT MENU]** You can choose to generate a report using the PREP report generation facility: +![PREP report menu](_static/images/PrepReportParameters.png)\ +By default, PREP generates a report after processing (report mode `'normal'`) and publishes it in PDF format +(`publishOn` is true). If you turn publishing off, PREP displays the report on the command line instead. +Choose `'skipReport'` to run the pipeline without a report, or `'reportOnly'` to return at a later step and +generate the report only. + +**[POSTPROCESS MENU]** After running the PREP pipeline, you can perform additional processing steps: +![PREP post process menu](_static/images/PrepPostProcess.png) + +**[SAVE MENU]** After PREP completes processing, reporting, and post processing, you are given the option of saving +the processed EEG data set: +![PREP final save menu](_static/images/PrepFinalSave.png) + +## PREP overview +This section discusses the algorithm and the meaning of the various parameters. + +### Processing steps +1. Handle boundary events prior to processing +1. Remove trend (high pass) temporarily to properly compute thresholds +1. Remove line noise without committing to a filtering strategy +1. Robustly reference the signal relative to an estimate of the "true" average reference +1. Detect and interpolate bad channels relative to this reference +1. Produce reports if desired +1. Post process if desired + +### Boundary marker handling +PREP is meant to work on data obtained from a continuous recording session. However, sometimes researchers record multiple sessions in the same file (such as by temporarily suspending and resuming recording). EEGLAB uses boundary events to mark +these discontinuities. Some (but not all) EEGLAB functions respect these boundary markers. By default, the PREP pipeline will not process data sets with boundary markers. Because some researchers use these boundary markers for other purposes than for marking discontinuities, PREP allows the option of temporarily removing boundary markers before processing and then reinserting afterwards. **You should not disregard boundary markers unless you are absolutely sure that these markers** +**to not represent discontinuities.** + +### Detrend (high pass filtering) +High pass filtering of some sort is needed in order for many of the algorithms, including line noise removal and +referencing to perform correctly. However, the exact cutoff may dramatically effect downstream algorithms. By default, +PREP uses a 1 Hz cutoff, but only temporarily filters, so that the final signal is not high-pass filtered. This +allows you to defer the final choice of high pass cutoff for downstream processing to later. + +#### Calling sequence for removing trends +The `removeTrend` function takes two structures in and produces two output structures. The `signal` structure +includes a `.data` field and an `.srate` field. The `signal` structure is compatible with an EEGLAB EEG structure, but does not rely on any of the other EEGLAB fields. The `.data` field should be channels x frames. + +As with all functions in the pipeline, the algorithm parameters are passed in a structure: +> `[signal, detrendOut] = removeTrend(signal)`\ +> `[signal, detrendOut] = removeTrend(signal, detrendIn)` + +The output structure contains all of the input structure fields plus additional fields including a string representation +of the actual command used. Usually, the only field that a user might need to provide is `detrendChannels` if the `signal` structure contains extra channels that represent items other than EEG signals. + +**Example:** +> `detrendIn = struct('detrendChannels', [1:32, 40:60], 'detrendCutoff', 0.5);` + +#### Parameters for removing the trend +The following parameters appear as fields in the `detrendIn` structure: + +**`detrendChannels`**\ + A row vector specifying the channel numbers of the channels to remove the trend from.\ +By default, PREP uses all of the channels (`1:size(signal.data, 1)`). +If your signal has extraneous or unused channels, you should specify which channels to use. + +**`detrendType`**\ +The type of detrending operation to perform. At this time the options are `'high pass'`, `'high pass sinc'`, `'linear'`, and +`'none'`. By default, PREP uses `'high pass'` by calling `pop_eegfiltnew` with the default settings. The +`'high pass sinc'` setting calls `pop_firws` with a Blackman window type. The `'linear'` filter is adapted +from the Chronux toolbox and uses local linear regression. The window size is `1.5/detrendCutoff`. Generally, +the `'linear'` option is much slower than simple high pass filtering and gives very similar results. +Usually you don't have to specify this parameter. + +**`detrendCutoff`**\ +The cutoff frequency in Hz for high pass filtering or local detrending. By default, PREP uses 1 Hz. Usually you don't have to specify this parameter. + +**`detrendStepSize`**\ +The amount in seconds to slide the local detrending window when local linear regression is used for detrending. +By default, PREP uses 0.02 seconds. This parameter is not used unless the `detrendType` is `'linear'`. + +### Line noise removal +We use an iterative version of a method that estimates the amplitude and size of a deterministic sinusoid at a specified frequency embedded in locally white noise. The model is applied in sliding windows to adjust for non stationarity. The algorithm requires a rough guess of the frequencies to be removed. By default, PREP uses multiples of 60 Hz. If the data set was recorded in a place where 50 Hz alternating current is used, you will need to provide the `lineFrequencies` parameter. Sometimes unusual frequencies appear due to aliasing and other recording artifacts. For example, a frequency spike at 212 Hz might appear as an aliasing artifact in a signal recorded at 512 Hz (212 = 512 - 300). You might need to rerun with different frequencies if unusual spectral peaks are visible in the reports. + +#### Calling sequence for line noise removal +The `removeLineNoise` function takes two structures in and produces two output structures. It fills in the +defaults for any parameters you leave out, checks them, and then calls `cleanLineNoise`, which does the work +and requires the full parameter structure. The `signal` structure +includes a `.data` field and an `.srate` field. The `signal` structure is compatible with an EEGLAB EEG structure, but does not rely on any of the other EEGLAB fields. The data field should be channels x frames. + +As with all functions in the pipeline, the algorithm parameters are passed in a structure. +> `[signal, lineNoiseOut] = removeLineNoise(signal)`\ +> `[signal, lineNoiseOut] = removeLineNoise(signal, lineNoiseIn)` + +The output structure contains all of the input structure fields plus additional fields containing information on the +tapers used to compute the spectral components and additional fields including a string representation +of the actual command used. + +**Example:** +> `lineNoiseIn = struct('lineNoiseChannels', [1:32, 40:60], 'lineFrequencies', [60, 120, 180, 212, 240]);` + +#### Parameters for line noise removal +The following parameters appear as fields in the `lineNoiseIn` structure: + +**`lineNoiseChannels`**\ + A row vector specifying the channel numbers of the channels to remove line noise from. By default, PREP uses all of the channels (`1:size(signal.data, 1)`). If your signal has extraneous or unused channels, you should specify which channels to use. + +**`Fs`**\ +The sampling frequency of the signal in Hz. By default, PREP uses the sampling rate specified in `signal.srate`. Usually you don't have to specify this parameter. + +**`lineFrequencies`**\ +A vector of frequencies in Hz of the approximate locations of the line noise peaks to remove. By default, PREP removes multiples of 60 Hz up to the Nyquist frequency (which is half of the sampling frequency). After looking at the spectrum in the report, you may need to redo PREP with additional frequencies. If the data was recorded in a location using a 50 Hz power, you will also need to override. + +The clean line noise procedure used in PREP can only remove sharp peaks with minimal spectral distortion. It does not remove broad peaks. If the PREP reports show that line noise has not been removed to a sufficient extent, you may have to perform additional filtering for a particular application. + +**`p`**\ +A significance cutoff level for removing a spectral peak. By default, PREP uses a p-value of 0.01. The clean line noise +procedure applies an F-test to determine whether a particular spectral peak is significantly higher than the background level in a small window. You should not have to override this parameter. + +**`fScanBandWidth`**\ +Half of the width of the frequency band centered on each line frequency. This band is used to search +for the exact value of the maximum amplitude frequency peak near the specified frequencies to be removed. +By default, PREP uses 2 Hz. You should not have to override this parameter. + +**`taperBandWidth`**\ +Bandwidth in Hz for the Sleppian tapers used to estimate the spectrum. By default, PREP uses 2 Hz. You should not have to override this parameter. + +**`taperWindowSize`**\ +Taper sliding window length in seconds. By default, PREP uses 4 seconds. You should not have to override this parameter. + +**`taperWindowStep`**\ +Taper sliding window step length in seconds. By default, PREP uses 1 second. You should not have to override this parameter. + +**`tau`**\ +The window overlap smoothing factor used in the exponent of the signmoidal smoothing functions. This sigmoidal smoothing function is used to patch results from sliding windows back together. By default, PREP uses a value of 100. You should not have to override this parameter. + +**`pad`**\ +Padding factor for FFTs (-1= no padding, 0 = pad to next power of 2, 1 = pad to power of two after, etc.). By default, PREP uses a pad factor of 0. A larger positive value gives better spectral results, but requires much greater computation time. Using a pad value of -1 is not recommended. You should not have to override this parameter. + +**`fPassBand`**\ +The frequency band (in units of Hz) used to compute the spectral background. By default, PREP +uses `[0, Fs/2]`. You may need to adjust this range to get better spectral estimates. + +**`maximumIterations`**\ +The maximum number of times that PREP applies the cleaning process to remove line noise. When a particular peak +is not significantly above the background, it is removed from consideration. When no significant peaks remain, PREP +stops the procedure. Most of the time, only a few iterations are required. You should not have to override this parameter. + +### Robust referencing +Referencing is the process of subtracting a common reference signal from all of the channels. Data sets collected +from Biosemi headsets require referencing of some sort. Other headsets benefit as well. When comparing results across +data sets, it is important to use the same referencing strategy. + +The PREP pipeline using robust average reference. This process is the same as average referencing (subtracting +the average of the channels from each channel in each frame) provided the data set does not have any bad +channels. However, if even if just a single channel has artifacts, the average reference can introduce +errors. To address this, the robust reference iteratively detects and interpolates bad channels to arrive at an +average reference that is not affected by artifacts. + +#### Calling sequence for referencing +The `performReference` function takes two structures in and produces two output structures. The signal structure +includes a `.data` field and an `.srate` field. The `signal` structure is compatible with an EEGLAB EEG structure, but does not rely on any of the other EEGLAB fields. The data field should be channels x frames. On output, PREP stores metadata about the referencing in the `.etc.noiseDetection` field. + +As with all functions in the pipeline, the algorithm parameters are passed in a structure: +> `[signal, referenceOut] = performReference(signal)`\ +> `[signal, referenceOut] = performReference(signal, referenceIn)` + +The output structure contains all of the input structure fields plus many additional fields containing the +reports of the output of the bad channel detection. Details in the document on PREP reporting. + +**Example:** +> `referenceIn = struct('referenceChannels', [1:32, 40:60]);` + +#### Parameters for referencing +The following parameters appear as fields in the `referenceIn` structure: + +**`referenceChannels`**\ + A row vector specifying the channel numbers of the channels to use for referencing. By default, PREP uses all of the channels (`1:size(signal.data, 1)`). If your signal has extraneous or unused channels, you should specify which channels to use. For standard robust referencing, you should specify only the EEG channels and not EOG channels or mastoids. All of the reference channels will be used to compute the reference. If the channel is bad, PREP uses its interpolated value. + +**`evaluationChannels`**\ + A row vector specifying the channel numbers of the channels to use for evaluating noisy channels. By default, PREP uses all of the channels (`1:size(signal.data, 1)`). If your signal has extraneous or unused channels, you should specify which channels to use. These channels should only be EEG channels. These channels are used to compute thresholds and to perform +estimates in the RANSAC algorithm. Often the reference channels and the evaluation channels are the same. However, if an EEG channel has NaNs or other unusable data, it will still be used as a reference channel, but will be excluded from the evaluation channels. + +**`rereferencedChannels`**\ + A row vector specifying the channel numbers of the channels from which to subtract the computed reference. By default, PREP uses all of the channels (`1:size(signal.data, 1)`). If your signal has extraneous or unused channels, you should specify which channels to use. Channels such as mastoids and EOG channels are usually re-referenced but are not used to +compute the robust reference. + +**`referenceType`**\ +The type of reference to be performed. By default, PREP uses `'robust'`, which computes an average reference with +iterative detection and interpolation of bad channels. Other options include `'average'`, `'specific'`, and `'none'`. +The `'average'` type removes the average of the reference channels with no interpolation, while `'specific'` removes +the average of the specified channels with no interpolation. If you mean to run the standardized PREP pipeline, +you don't need to specify this field. + +**`interpolationOrder`**\ +Specifies whether PREP performs final channel interpolation. By default, PREP uses `'post-reference'`. +In this case, after a final robust reference is computed, the channels are re-interpolated and the reference is corrected. +In `'pre-reference'`, PREP incrementally adds to the bad channel list and interpolates before computing the reference. +If the initial estimate of the reference is poor, this is not a good approach. In the `'none'` option, PREP removes the +reference but does not interpolate. Bad channels remain in the signal. You may choose to remove them during post-processing. If you mean to run the standardized PREP pipeline, you don't need to specify this field. + +**`meanEstimateType`**\ +The method used to estimate the initial mean reference. By default, PREP takes the median of the channel values +in each frame. Other options include `'mean'`, which is prone to outliers, `'huber'` which is computationally expensive, +or `'none'`. If you mean to run the standardized PREP pipeline, you don't need to specify this field. + +**`channelLocations`**\ +A structure containing the channel locations in EEGLAB `chanlocs` format. By default, PREP uses the `signal.chanlocs` +structure unless this field is used to over ride. PREP must have channel locations in order to work. + +**`channelInformation`**\ +A structure containing channel information in EEGLAB `chaninfo` format. By default, PREP uses the `signal.chaninfo` +structure unless this field is used to over ride. PREP uses the nose direction for display purposes in the reports. + +**`srate`**\ +The sampling frequency of the signal in Hz. By default, PREP uses the sampling rate specified in `signal.srate`. Usually you don't have to specify this parameter. + +**`samples`**\ +The number of frames to use for the computation. By default, PREP uses `size(signal.data, 2)`. Usually you don't have to specify this parameter. + +**`robustDeviationThreshold`**\ +Z-score cutoff for robust channel deviation. If a channel has a robust deviation z-score above this value, +PREP considers the channel to be bad in that window. By default, PREP uses 5. Usually you don't have to specify this parameter. + +**`highFrequencyNoiseThreshold`**\ +Z-score cutoff for SNR (signal above 50 Hz). If a channel has a z-score of the ratio of signal above 50 Hz to that below 50 Hz, PREP considers the channel to be bad in that window. By default, PREP uses 5. Usually you don't have to specify this parameter. + +**`correlationWindowSeconds`**\ +Window size in seconds for computing correlations and other window values. By default, PREP uses 1. Usually you don't have to specify this parameter. + +**`correlationThreshold`**\ +Max correlation absolute threshold for channel being bad in a window. In each window, PREP computes the maximum of the absolute value of the correlation with other channels and compares to this threshold. If the correlation falls below this threshold, PREP considers the channel to be bad-by-correlation in this window. PREP also uses this window size to evaluate windowed absolute deviation and SNR. By default, PREP uses 0.4. Usually you don't have to specify this parameter. + +**`badTimeThreshold`**\ +Threshold fraction of bad correlation windows for designating a channel to be bad-by-correlation. By default, PREP uses 0.01. Usually you don't have to specify this parameter. + +**`ransacOff`**\ +If true, RANSAC is not used for bad channel detection (useful for small headsets). By default, PREP uses false. Usually you don't have to specify this parameter. + +**`ransacSampleSize`**\ +Number of random matrices sampled to estimate RANSAC. By default, PREP uses 50 sample matrices. Usually you don't have to specify this parameter. + +**`ransacChannelFraction`**\ +Fraction of evaluation channels RANSAC uses to predict a channel. By default, PREP uses 0.25 of the evaluation channels. Usually you don't have to specify this parameter. + +**`ransacCorrelationThreshold`**\ +Cutoff correlation for unpredictability by neighbors. If the absolute correlation of the channel with its RANSAC prediction in a window falls below this threshold, the channel is designated as bad in this window. By default, PREP uses 0.75. Usually you don't have to specify this parameter. + +**`ransacUnbrokenTime`**\ +Threshold fraction of windows that a channel must be bad before it is designated as a channel that is bad-by-RANSAC. By default, PREP uses 0.4. Usually you don't have to specify this parameter. + +**`ransacWindowSeconds`**\ +Size of windows in seconds over which to compute RANSAC predictions. By default, PREP uses 5. Usually you don't have to specify this parameter. + +**`maxReferenceIterations`**\ +Maximum number of iterations in the reference-bad channel detection-interpolation cycle. +By default, PREP uses 4. If the actual iterations is 4, you may need to increase this or look carefully at your data set. + +**`reportingLevel`**\ +How much information to store about referencing in the EEG structure. By default, PREP uses `'verbose'`, which causes +the data structure to contain all of the window information for later processing and reporting. If you use `'minimum'`, you will not be able to run the PREP reports. Alternatively, you can choose to clean up the data structure +after running the reports. + +### Reporting +PREP has an extensive report facility that can be used provided that your reporting level was `'verbose'`. The GUI version of the PREP pipeline (`pop_prepPipeline`) has options in the report GUI for you to select whether or not to run the report. If the report mode is `'normal'` (the default), then PREP runs the processing pipeline followed by the report, followed by the post processing. If the report mode is `'skipReport'`, then PREP runs the processing pipeline followed by the post processing. If the report mode is `'reportOnly'`, then PREP only runs the report and skips both the processing and the post processing. + +#### Calling sequence for reporting +The `publishPrepReport` function takes an EEG structure that has been run through the PREP pipeline with +report level of verbose. You need to furnish a summary directory name and a summary file name for an HTML file with summary information for all of the stuff. + +As with all functions in the pipeline, the algorithm parameters are passed in a structure. +> `publishPrepReport(signal, summaryFilePath, sessionFilePath, consoleFID, publishOn);` + +`publishPrepReport` returns nothing; its results are files. It appends a summary of this dataset to the HTML file +at `summaryFilePath`, whether or not `publishOn` is true, and when `publishOn` is true it publishes the detailed +report as a PDF at `sessionFilePath`. + +**Example:**\ +The following produces an HTML-formatted summary report in the current directory and publishes a detailed report in the s1 sub directory. +> `publishPrepReport(EEG, 'vepSummary.html', '.\s1\vep01.pdf', 1, true);` + +#### Parameters for reporting + +**`signal`**\ +The `signal` structure includes a `.data` field and an `.srate` field. The `signal` structure is compatible with an EEGLAB EEG structure, but does not rely on any of the other EEGLAB fields. The data field should be channels x frames. In order to get reports, the EEG structure must have the `.etc.noiseDetection` as PREP generates the report from information stored there. + +**`summaryFilePath`**\ +The file name for the HTML summary file that PREP produces for the report. The name should include path information when needed. PREP appends the summary for this dataset to this file whether or not `publishOn` is `true`, so calling it for each dataset in a collection builds one collection summary. `consoleFID` receives the console output, not the summary. + +**`sessionFilePath`**\ +The file name for the detailed PDF report that PREP produces. The name should include path information when needed. If `publishOn` is `false`, PREP doesn't produce a report. + +**`consoleFID`**\ +An open file descriptor for writing reporting information. Usually, this is 1, indicating that output should be directed to the command window. Give an open file descriptor to another file to record the report in a log. + +**`publishOn`**\ +If `true` (the default) PREP produces a published PDF Report and an HTML summary. If `false`, the PREP runs reporting, but keeps the figures displayed and outputs the reporting information to the command window. This mode is useful for closer examination of the figures. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e805da3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +# This repository is MATLAB code. pyproject.toml exists only to declare the +# Python toolchain that builds the documentation in docs/. It is not a Python +# package and is never published; the PREP version lives in +# PrepPipeline/utilities/getPrepVersion.m, and docs/conf.py reads it from there. + +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "eeg-clean-tools-docs" +version = "0.0.0" +description = "Documentation toolchain for the PREP pipeline (MATLAB)" +readme = "README.md" +requires-python = ">=3.10" +license = {text = "GPL-2.0-or-later"} + +[project.urls] +Homepage = "https://github.com/VisLab/EEG-Clean-Tools" +Documentation = "https://vislab.github.io/EEG-Clean-Tools/" +"Bug Tracker" = "https://github.com/VisLab/EEG-Clean-Tools/issues" + +[project.optional-dependencies] +docs = [ + "sphinx>=7.1.0,<10.0", + "furo>=2024.1.29", + "sphinx-copybutton>=0.5.2", + "myst-parser>=3.0.0", + "sphinxcontrib-matlabdomain==0.22.1", + "linkify-it-py>=2.0.3", +] + +[tool.setuptools] +packages = []