Skip to content

Commit 21f904b

Browse files
vncntprvstvncntprvst
committed
container runs - pyks fails at datashift2
1 parent 0aacd3b commit 21f904b

6 files changed

Lines changed: 117 additions & 12 deletions

File tree

‎pykilosort/README.md‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,39 @@
11
### Build this image
2+
Build default image:
3+
docker build -t pykilosort:latest -f dockerfiles/Dockerfile .
24

5+
Build image with additional testing components:
6+
docker build -t pykilosort:test -f dockerfiles/Dockerfile.testing context
37

48
### Run container
5-
docker run --rm --gpus all pykilosort
9+
docker run --rm -it -v <host-data-folder>:<docker-data-folder> --gpus all pykilosort:latest
10+
flags:
11+
--rm: removes container once it's stopped
12+
-it: for interactive session
13+
-v: mounted volumes (directories)
14+
--gpus: enables GPU use within container
15+
16+
### Test
17+
Download Neuropixel 1.0 data to your data directory: https://catalystneuro.github.io/spike-sorting-hackathon/datasets/datasets.html#allen-institute-example
18+
(see also https://github.com/int-brain-lab/pykilosort/tree/ibl_prod/examples, although apparently not up to date)
19+
20+
$docker run --rm -it -v /my/dir/to/data:/data --gpus all pykilosort
21+
#conda activate pyks2
22+
#cd /data
23+
--- alternatively, get data from kachery (if installed and configured) ---
24+
```
25+
wget https://catalystneuro.github.io/spike-sorting-hackathon/datasets/examples/example_allen_NP1.py
26+
python example_allen_NP1.py
27+
```
28+
29+
The either run tests in ipython console, or run example from /home/test_file directory (if using `.testing` image), after editing directory paths
30+
#ipython
31+
```
32+
from pathlib import Path
33+
from pykilosort import run, add_default_handler, np1_probe, np2_probe
34+
35+
data_path = Path('/data/Allen_Institute_NP1/continuous_1min.bin')
36+
dir_path = Path('/data/Allen_Institute_NP1/pyKS_output')
37+
add_default_handler(level='INFO') # print output as the algorithm runs
38+
run(data_path, dir_path=dir_path, probe=np1_probe())
39+
```

‎pykilosort/build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
#!/bin/bash
22

3-
docker build -t spikeinterface/pykilosort:0.1.0 .
3+
docker build -t spikeinterface/pykilosort:latest -t spikeinterface/pykilosort:0.1.0 .

‎pykilosort/context/packages/testing_requirements.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
spikeinterface[full]
2+
dandi
3+
kachery
4+
kachery-cloud
5+
git+https://github.com/flatironinstitute/spikeforest2.git@master

‎pykilosort/context/test_files/example_general_users.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
from pathlib import Path
2+
from pykilosort import run, add_default_handler, np1_probe, np2_probe
3+
4+
# Run standard ks2.5 algorithm for a np1 probe
5+
data_path = Path('path/to/data/data.bin')
6+
dir_path = Path('path/to/output/folder') # by default uses the same folder as the dataset
7+
add_default_handler(level='INFO') # print output as the algorithm runs
8+
run(data_path, dir_path=dir_path, probe=np1_probe())
9+
10+
# Run chronic recordings for a np2 probe
11+
# For now this still uses ks2.5 clustering, chronic clustering algorithm coming soon!
12+
data_paths = [
13+
Path('path/to/first/dataset/dataset.bin'),
14+
Path('path/to/second/dataset/dataset.bin'),
15+
Path('path/to/third/dataset/dataset.bin'),
16+
]
17+
dir_path = Path('path/to/output/folder') # by default uses the same folder as the first dataset
18+
add_default_handler(level='INFO')
19+
run(data_paths, dir_path=dir_path, probe=np2_probe(), low_memory=True)

pykilosort/Dockerfile renamed to pykilosort/dockerfiles/Dockerfile

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ LABEL maintainer="Vincent Prevosto <prevosto@mit.edu>"
44

55
# USER root
66
# Ubuntu package installs
7-
RUN apt update
8-
RUN apt install -y --no-install-recommends \
7+
RUN apt update && \
8+
apt install -y --no-install-recommends \
99
libfftw3-dev \
1010
git \
1111
wget && \
@@ -20,7 +20,7 @@ ENV LATEST_CONDA_SCRIPT "Miniconda3-py38_$MINICONDA_VERSION-Linux-x86_64.sh"
2020
RUN wget --quiet https://repo.anaconda.com/miniconda/$LATEST_CONDA_SCRIPT -O ~/miniconda.sh && \
2121
bash ~/miniconda.sh -b -p $CONDA_DIR && \
2222
rm ~/miniconda.sh
23-
# RUN echo 'export PATH="~/miniconda/bin:$PATH"' >> ~/.bashrc
23+
# RUN echo 'export PATH="/home/miniconda3/bin:$PATH"' >> ~/.bashrc
2424
ENV PATH=$CONDA_DIR/bin:$PATH
2525
RUN conda update conda && \
2626
conda install conda-build
@@ -39,10 +39,6 @@ RUN conda env create -f pyks2.yml
3939

4040
# Install pykilosort
4141
SHELL ["conda","run","-n","pyks2","/bin/bash","-c"]
42-
# RUN conda install --quiet --yes ipykernel && \
43-
# python -m ipykernel install --user --name pyks2 --display-name "pyKilosort" && \
44-
RUN conda develop .
45-
46-
47-
48-
42+
RUN conda install --quiet --yes ipykernel && \
43+
python -m ipykernel install --user --name pyks2 --display-name "pyKilosort" && \
44+
conda develop .

‎pykilosort/dockerfiles/Dockerfile.testing‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
FROM nvidia/cuda:10.0-base-ubuntu18.04
2+
3+
LABEL maintainer="Vincent Prevosto <prevosto@mit.edu>"
4+
5+
# Copy files for dev/tests
6+
COPY packages /srv/packages
7+
COPY test_files /home/test_files
8+
9+
# USER root
10+
# Ubuntu package installs
11+
RUN apt update && \
12+
apt install -y --no-install-recommends \
13+
libfftw3-dev \
14+
git \
15+
wget && \
16+
apt clean && \
17+
rm -rf /var/lib/apt/lists/*
18+
19+
# install miniconda
20+
ENV MINICONDA_VERSION 4.12.0
21+
ENV CONDA_DIR /home/miniconda3
22+
ENV LATEST_CONDA_SCRIPT "Miniconda3-py38_$MINICONDA_VERSION-Linux-x86_64.sh"
23+
24+
RUN wget --quiet https://repo.anaconda.com/miniconda/$LATEST_CONDA_SCRIPT -O ~/miniconda.sh && \
25+
bash ~/miniconda.sh -b -p $CONDA_DIR && \
26+
rm ~/miniconda.sh
27+
# RUN echo 'export PATH="/home/miniconda3/bin:$PATH"' >> ~/.bashrc
28+
ENV PATH=$CONDA_DIR/bin:$PATH
29+
RUN conda update conda && \
30+
conda install conda-build
31+
32+
# make conda activate command available from /bin/bash --login shells
33+
RUN echo ". $CONDA_DIR/etc/profile.d/conda.sh" >> /root/.profile
34+
# make conda activate command available from /bin/bash --interactive shells
35+
RUN conda init bash
36+
37+
# Install IBL port of pykilosort
38+
RUN git clone -b drift_test_stable https://github.com/kushbanga/pykilosort.git /src/pykilosort
39+
WORKDIR /src/pykilosort
40+
41+
# Create environment
42+
RUN conda env create -f pyks2.yml
43+
44+
# Install pykilosort
45+
SHELL ["conda","run","-n","pyks2","/bin/bash","-c"]
46+
RUN conda install --quiet --yes ipykernel && \
47+
python -m ipykernel install --user --name pyks2 --display-name "pyKilosort" && \
48+
pip install -U -r /srv/packages/testing_requirements.txt && \
49+
conda develop .
50+
51+
RUN rm -rf /srv/packages

0 commit comments

Comments
 (0)