Skip to content

Commit 3fb146b

Browse files
leoankleoank
committed
refactor(starrynight): merge pipeline steps
partially solves #9 #10 Some issues with MeasureColocalization is remaining
1 parent f426f7b commit 3fb146b

30 files changed

+591
-72
lines changed

‎nix/linkml.nix

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
lib,
3+
buildPythonPackage,
4+
fetchPypi,
5+
rustPlatform,
6+
}:
7+
buildPythonPackage rec {
8+
pname = "linkml";
9+
version = "1.8.7";
10+
pyproject = true;
11+
12+
src = fetchPypi {
13+
inherit pname version;
14+
sha256 = "";
15+
16+
};
17+
18+
dependencies = [
19+
];
20+
pythonImportsCheck = [ "cpgparser" ];
21+
22+
meta = {
23+
description = "CPG parser python library";
24+
homepage = "https://github.com/broadinstitute/cpg";
25+
license = lib.licenses.mit;
26+
};
27+
28+
}

‎starrynight/notebooks/__marimo__/session/exec_pipeline_001.py.json

+163
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
{
2+
"version": "1",
3+
"metadata": {
4+
"marimo_version": "0.11.30"
5+
},
6+
"cells": [
7+
{
8+
"id": "Hbol",
9+
"code_hash": "b434a9967ea06bb56492f5e8391d52f2",
10+
"outputs": [
11+
{
12+
"type": "data",
13+
"data": {
14+
"text/plain": ""
15+
}
16+
}
17+
],
18+
"console": []
19+
},
20+
{
21+
"id": "kWhL",
22+
"code_hash": null,
23+
"outputs": [
24+
{
25+
"type": "data",
26+
"data": {
27+
"text/plain": ""
28+
}
29+
}
30+
],
31+
"console": []
32+
},
33+
{
34+
"id": "arBd",
35+
"code_hash": null,
36+
"outputs": [
37+
{
38+
"type": "data",
39+
"data": {
40+
"text/plain": ""
41+
}
42+
}
43+
],
44+
"console": []
45+
},
46+
{
47+
"id": "EIPH",
48+
"code_hash": null,
49+
"outputs": [
50+
{
51+
"type": "data",
52+
"data": {
53+
"text/html": "<pre style='font-size: 12px'>DataConfig(dataset_path=PosixPath(&#x27;/datastore/ip-merck-dev/projects/cpg1234-AMD-screening&#x27;), storage_path=PosixPath(&#x27;/datastore/ip-merck-dev/projects/cpg1234-AMD-screening&#x27;), workspace_path=PosixPath(&#x27;/datastore/ip-merck-dev/projects/cpg1234-AMD-screening/workspace&#x27;))</pre>"
54+
}
55+
}
56+
],
57+
"console": []
58+
},
59+
{
60+
"id": "MJUe",
61+
"code_hash": "5d11b2d0a914598a08672a3f1a2f5bf7",
62+
"outputs": [
63+
{
64+
"type": "data",
65+
"data": {
66+
"text/plain": ""
67+
}
68+
}
69+
],
70+
"console": []
71+
},
72+
{
73+
"id": "vblA",
74+
"code_hash": "5970236690af795b8a77278c3dd03e7b",
75+
"outputs": [
76+
{
77+
"type": "data",
78+
"data": {
79+
"text/plain": ""
80+
}
81+
}
82+
],
83+
"console": []
84+
},
85+
{
86+
"id": "bkHC",
87+
"code_hash": "e576fc987c84c793f545c1f924eab999",
88+
"outputs": [
89+
{
90+
"type": "data",
91+
"data": {
92+
"text/plain": ""
93+
}
94+
}
95+
],
96+
"console": [
97+
{
98+
"type": "stream",
99+
"name": "stdout",
100+
"text": "<pipecraft.pipeline.Seq object at 0x7efc6cd12310>\n"
101+
}
102+
]
103+
},
104+
{
105+
"id": "lEQa",
106+
"code_hash": null,
107+
"outputs": [
108+
{
109+
"type": "error",
110+
"ename": "exception",
111+
"evalue": "The variable 'gen_inv_mod' was defined by another cell",
112+
"traceback": []
113+
}
114+
],
115+
"console": [
116+
{
117+
"type": "stream",
118+
"name": "stdout",
119+
"text": "GenInvModule(spec=Container(inputs=[TypeInput(name='dataset_path', type='files', description='Path to the dataset.', optional=False, path='/datastore/ip-merck-dev/projects/cpg1234-AMD-screening', format=None, collection=None, subtype=None, z=None, t=None, tiled=None, pyramidal=None)], outputs=[TypeOutput(name='project_inventory', type='file', description='Generated Inventory', optional=False, path='/datastore/ip-merck-dev/projects/cpg1234-AMD-screening/workspace/inventory/inventory.parquet', format=None, collection=None, subtype=None, z=None, t=None, tiled=None, pyramidal=None)], parameters=[], display_only=[], results=[], exec_function=ExecFunction(name='', script='', module='', cli_command='', hidden_args=None), docker_image=None, algorithm_folder_name=None, citations=TypeCitations(algorithm=[TypeAlgorithmFromCitation(name='Starrynight inventory module', doi=None, license=None, description='This module generates an inventory for the dataset.')])), pipe=<pipecraft.pipeline.Seq object at 0x7efce65bdd90>, uow=[UnitOfWork(inputs={}, outputs={'inventory': ['/datastore/ip-merck-dev/projects/cpg1234-AMD-screening/workspace/inventory/inventory.parquet']})])\n"
120+
}
121+
]
122+
},
123+
{
124+
"id": "FLbe",
125+
"code_hash": null,
126+
"outputs": [
127+
{
128+
"type": "data",
129+
"data": {
130+
"text/plain": ""
131+
}
132+
}
133+
],
134+
"console": []
135+
},
136+
{
137+
"id": "GDGc",
138+
"code_hash": null,
139+
"outputs": [
140+
{
141+
"type": "data",
142+
"data": {
143+
"text/plain": ""
144+
}
145+
}
146+
],
147+
"console": []
148+
},
149+
{
150+
"id": "zHer",
151+
"code_hash": "88e5e46bc50e5c3e38b048248417468a",
152+
"outputs": [
153+
{
154+
"type": "data",
155+
"data": {
156+
"text/html": "<span class=\"markdown prose dark:prose-invert\"><h2 id=\"setting-up-dataset-path\">Setting up dataset path</h2></span>"
157+
}
158+
}
159+
],
160+
"console": []
161+
}
162+
]
163+
}

‎starrynight/notebooks/exec_pcp_generic_pipe.py

+104-9
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,11 @@
77

88
from starrynight.experiments.pcp_generic import PCPGeneric, PCPGenericInitConfig
99

10-
# cp_illum_apply
11-
from starrynight.modules.cp_illum_apply.apply_cp import (
12-
CPApplyIllumInvokeCPModule,
13-
)
14-
from starrynight.modules.cp_illum_apply.apply_cppipe import (
15-
CPApplyIllumGenCPPipeModule,
16-
)
10+
# from starrynight.modules.analysis.analysis_cp import AnalysisInvokeCPModule
11+
# from starrynight.modules.analysis.analysis_cppipe import AnalysisGenCPPipeModule
12+
# from starrynight.modules.analysis.analysis_load_data import AnalysisGenLoadDataModule
13+
from starrynight.modules.cp_illum_apply.apply_cp import CPApplyIllumInvokeCPModule
14+
from starrynight.modules.cp_illum_apply.apply_cppipe import CPApplyIllumGenCPPipeModule
1715
from starrynight.modules.cp_illum_apply.apply_load_data import (
1816
CPApplyIllumGenLoadDataModule,
1917
)
@@ -40,7 +38,29 @@
4038
from starrynight.modules.gen_index import GenIndexModule
4139
from starrynight.modules.gen_inv import GenInvModule
4240

43-
# schema
41+
# from starrynight.modules.sbs_align.algin_cp import SBSAlignInvokeCPModule
42+
# from starrynight.modules.sbs_align.algin_cppipe import SBSAlignGenCPPipeModule
43+
# from starrynight.modules.sbs_align.algin_load_data import SBSAlignGenLoadDataModule
44+
from starrynight.modules.sbs_illum_apply.apply_cp import SBSApplyIllumInvokeCPModule
45+
from starrynight.modules.sbs_illum_apply.apply_cppipe import (
46+
SBSApplyIllumGenCPPipeModule,
47+
)
48+
from starrynight.modules.sbs_illum_apply.apply_load_data import (
49+
SBSApplyIllumGenLoadDataModule,
50+
)
51+
from starrynight.modules.sbs_illum_calc.calc_cp import SBSCalcIllumInvokeCPModule
52+
from starrynight.modules.sbs_illum_calc.calc_cppipe import SBSCalcIllumGenCPPipeModule
53+
from starrynight.modules.sbs_illum_calc.calc_load_data import (
54+
SBSCalcIllumGenLoadDataModule,
55+
)
56+
57+
# from starrynight.modules.sbs_preprocess.preprocess_cp import SBSPreprocessInvokeCPModule # noqa: E501
58+
# from starrynight.modules.sbs_preprocess.preprocess_cppipe import (
59+
# SBSPreprocessGenCPPipeModule,
60+
# )
61+
# from starrynight.modules.sbs_preprocess.preprocess_load_data import (
62+
# SBSPreprocessGenLoadDataModule,
63+
# )
4464
from starrynight.schema import DataConfig
4565

4666
# %% [markdown]
@@ -101,7 +121,7 @@
101121
barcode_csv_path="",
102122
cp_acquisition_order="snake",
103123
cp_img_frame_type="round",
104-
cp_img_overlap_pct="10",
124+
cp_img_overlap_pct=10,
105125
sbs_acquisition_order="snake",
106126
sbs_img_frame_type="round",
107127
sbs_img_overlap_pct="10",
@@ -252,3 +272,78 @@
252272
exec_mounts,
253273
)
254274
run = exec_backend.run()
275+
276+
# ------------------------------------------------------------------
277+
278+
# %% [markdown]
279+
# Step 5: SBS calculate illum correction
280+
281+
# %%
282+
# Gen load data
283+
sbs_calc_illum_load_data_mod = SBSCalcIllumGenLoadDataModule.from_config(
284+
data_config, pcp_experiment
285+
)
286+
287+
exec_backend = SnakeMakeBackend(
288+
sbs_calc_illum_load_data_mod.pipe, backend_config, exec_runs / "run012", exec_mounts
289+
)
290+
run = exec_backend.run()
291+
292+
# Gen cppipe file
293+
sbs_calc_illum_cppipe_mod = SBSCalcIllumGenCPPipeModule.from_config(
294+
data_config, pcp_experiment
295+
)
296+
297+
exec_backend = SnakeMakeBackend(
298+
sbs_calc_illum_cppipe_mod.pipe, backend_config, exec_runs / "run013", exec_mounts
299+
)
300+
run = exec_backend.run()
301+
302+
# Invoke cppipe file
303+
sbs_calc_illum_invoke_mod = SBSCalcIllumInvokeCPModule.from_config(
304+
data_config, pcp_experiment
305+
)
306+
307+
exec_backend = SnakeMakeBackend(
308+
sbs_calc_illum_invoke_mod.pipe, backend_config, exec_runs / "run014", exec_mounts
309+
)
310+
run = exec_backend.run()
311+
312+
# ------------------------------------------------------------------
313+
314+
# %% [markdown]
315+
# Step 6: SBS apply illum correction
316+
317+
# %%
318+
# Gen load data
319+
sbs_apply_illum_load_data_mod = SBSApplyIllumGenLoadDataModule.from_config(
320+
data_config, pcp_experiment
321+
)
322+
323+
exec_backend = SnakeMakeBackend(
324+
sbs_apply_illum_load_data_mod.pipe,
325+
backend_config,
326+
exec_runs / "run015",
327+
exec_mounts,
328+
)
329+
run = exec_backend.run()
330+
331+
# Gen cppipe file
332+
sbs_apply_illum_cppipe_mod = SBSApplyIllumGenCPPipeModule.from_config(
333+
data_config, pcp_experiment
334+
)
335+
336+
exec_backend = SnakeMakeBackend(
337+
sbs_apply_illum_cppipe_mod.pipe, backend_config, exec_runs / "run016", exec_mounts
338+
)
339+
run = exec_backend.run()
340+
341+
# Invoke cppipe file
342+
sbs_apply_illum_invoke_mod = SBSApplyIllumInvokeCPModule.from_config(
343+
data_config, pcp_experiment
344+
)
345+
346+
exec_backend = SnakeMakeBackend(
347+
sbs_apply_illum_invoke_mod.pipe, backend_config, exec_runs / "run017", exec_mounts
348+
)
349+
run = exec_backend.run()

0 commit comments

Comments
 (0)