Nextflow CPU limit does not take effect

[Jasonqte]

Hi,

I am running my program on Nextflow version 24.10.5. The content of my nextflow.config is as follows:

params {

  // Workflow flags
  // TODO nf-core: Specify your pipeline's command line flags
  genome      = "GRCh38"
  input       = null
  input_csv   = "/workspace/ieee/CancerDataset_phase2_sample/GSM8462129/GSM8462129.csv"
  single_end  = false
  outdir = '/workspace/ieee/CancerDataset_phase2_sample/GSM8462129/starsolo_v2/'
  publish_dir_mode = 'copy'

  // UMI tools options
  umitools_bc_pattern = "CCCCCCCCCCCCCCCCNNNNNNNNNN"
  save_umi_intermeds = true
  skip_umitools = true

  // Alignment options
  skip_star            = true
  min_mapped_reads     = 5
  save_align_intermeds = true
  star_ignore_sjdbgtf  = false

  // Inputs for SICILIAN after STAR alignment
  star_bam               = true
  star_sj_out_tab        = true
  reads_per_gene         = true
  star_chimeric_junction = true

  // Sicilian configs
  skip_classinput = true
  skip_glm        = false
  smartseq2       = false
  tenx            = true
  stranded        = true
  domain          = "/workspace/ieee/Project/alternative_splicing/nf-sicilian/SICILIAN_human_hg38_Refs/domain_file/ucscGenePfam.txt"
  annotator       = "/workspace/ieee/Project/alternative_splicing/nf-sicilian/SICILIAN_human_hg38_Refs/annotator_file/hg38_refseq.pkl"
  exon_bounds     = "/workspace/ieee/Project/alternative_splicing/nf-sicilian/SICILIAN_human_hg38_Refs/exon_pickle_file/hg38_refseq_exon_bounds.pkl"
  splices         = "/workspace/ieee/Project/alternative_splicing/nf-sicilian/SICILIAN_human_hg38_Refs/splice_pickle_file/hg38_refseq_splices.pkl"
  // Inputs for SICILIAN after class_input (e.g. if ran partially and restarting)
  sicilian_class_input = false
  // Inputs for SICILIAN after GLM (e.g. if ran partially and restarting)
  sicilian_glm_output = false

  // Genome inputs
  star_index = "/workspace/ieee/Project/alternative_splicing/starsolo/genome_100/"
  gtf = "/workspace/ieee/Project/alternative_splicing/starsolo/refdata-gex-GRCh38-2024-A/genes/genes.gtf"
  save_reference = true

  // MultiQC options
  skip_multiqc = false
  multiqc_title              = null

  // Boilerplate options
  seq_center = false
  enable_conda = false
  multiqc_config = false
  email = false
  email_on_fail = false
  max_multiqc_email_size = 25.MB
  plaintext_email = false
  monochrome_logs = false
  help = false
  igenomes_base = '/workspace/ieee/Project/alternative_splicing/nf-sicilian'
  tracedir = "${params.outdir}/pipeline_info"
  igenomes_ignore = false
  custom_config_version = 'master'
  custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}"
  hostnames = false
  config_profile_name = false
  config_profile_description = false
  config_profile_contact = false
  config_profile_url = false
  validate_params = true
  show_hidden_params = false
  schema_ignore_params = 'genomes,input_paths,modules,sicilian_glm_output_paths,sicilian_glm_output,sicilian_class_input_paths,sicilian_class_input,star_chimeric_junction_paths,star_reads_per_gene_paths,star_sj_out_tab_paths,star_bam_paths,star_chimeric_junction,reads_per_gene,star_sj_out_tab,star_bam'

  // Defaults only, expecting to be overwritten
  max_memory = 1024.GB
  max_cpus = 70
  max_time = 24.h

}

process {
    withName: STAR_ALIGN {  
        cpus   = 60       
        memory = 1024.GB 
        time   = 4.h    
    }
    withName: SICILIAN:SICILIAN_CLASSINPUT {  
        cpus   = 60       
        memory = 1024.GB
        time   = 8.h    
    }
    withName: SICILIAN:SICILIAN_GLM {  
        cpus   = 60       
        memory = 1024.GB 
        time   = 8.h    
    }
    withName: SICILIAN:SICILIAN_ANNSPLICES {  
        cpus   = 60       
        memory = 1024.GB 
        time   = 8.h    
    }
    withName: SICILIAN:SICILIAN_CONSOLIDATE {  
        cpus   = 60       
        memory = 1024.GB 
        time   = 8.h    
    }
    withName: SICILIAN:SICILIAN_PROCESS_CI_10X {  
        cpus   = 60       
        memory = 1024.GB 
        time   = 8.h    
    }
    withName: SICILIAN:SICILIAN_POSTPROCESS {  
        cpus   = 60       
        memory = 1024.GB 
        time   = 8.h    
    }
    withName: MULTIQC {  
        cpus   = 60       
        memory = 1024.GB 
        time   = 8.h    
    }
}
// Container slug. Stable releases should specify release tag!
// Developmental code should specify :dev
// process.container = 'nfcore/sicilian:dev'

// Load base.config by default for all pipelines
includeConfig 'conf/base.config'

// Load modules.config for DSL2 module specific options
includeConfig 'conf/modules.config'


// Load nf-core custom profiles from different Institutions
try {
  includeConfig "${params.custom_config_base}/nfcore_custom.config"
} catch (Exception e) {
  System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")
}

profiles {
  conda {
    docker.enabled = false
    singularity.enabled = false
    podman.enabled = false
    shifter.enabled = false
    charliecloud = false
    process.conda = "$projectDir/environment.yml"
  }
  debug { process.beforeScript = 'echo $HOSTNAME' }
  docker {
    docker.enabled = true
    singularity.enabled = false
    podman.enabled = false
    shifter.enabled = false
    charliecloud.enabled = false
    // Avoid this error:
    //   WARNING: Your kernel does not support swap limit capabilities or the cgroup is not mounted. Memory limited without swap.
    // Testing this in nf-core after discussion here https://github.com/nf-core/tools/pull/351
    // once this is established and works well, nextflow might implement this behavior as new default.
    docker.runOptions = '-u \$(id -u):\$(id -g)'
  }
  singularity {
    docker.enabled = false
    singularity.enabled = true
    podman.enabled = false
    shifter.enabled = false
    charliecloud.enabled = false
    singularity.autoMounts = true
  }
  podman {
    singularity.enabled = false
    docker.enabled = false
    podman.enabled = true
    shifter.enabled = false
    charliecloud = false
  }
  shifter {
    singularity.enabled = false
    docker.enabled = false
    podman.enabled = false
    shifter.enabled = true
    charliecloud.enabled = false
  }
  charliecloud {
    singularity.enabled = false
    docker.enabled = false
    podman.enabled = false
    shifter.enabled = false
    charliecloud.enabled = true
  }
  test { includeConfig 'conf/test.config' }
  test_samplesheet { includeConfig 'conf/test_samplesheet.config' }
  test_skip_umitools { includeConfig 'conf/test_skip_umitools.config' }
  test_smartseq2 { includeConfig 'conf/test_smartseq2.config' }
  test_full { includeConfig 'conf/test_full.config' }
}

// Load igenomes.config if required
if (!params.igenomes_ignore) {
  includeConfig 'conf/igenomes.config'
}

// Export these variables to prevent local Python/R libraries from conflicting with those in the container
env {
  PYTHONNOUSERSITE = 1
  R_PROFILE_USER = "/.Rprofile"
  R_ENVIRON_USER = "/.Renviron"
}

// Capture exit codes from upstream processes when piping
process.shell = ['/bin/bash', '-euo', 'pipefail']

def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')
timeline {
    enabled = true
    file    = "${params.tracedir}/execution_timeline_${trace_timestamp}.html"
}
report {
    enabled = true
    file    = "${params.tracedir}/execution_report_${trace_timestamp}.html"
}
trace {
    enabled = true
    file    = "${params.tracedir}/execution_trace_${trace_timestamp}.txt"
}
dag {
    enabled = true
    file    = "${params.tracedir}/pipeline_dag_${trace_timestamp}.svg"
}

manifest {
  name = 'nf-core/sicilian'
  author = 'Julia Salzman Lab'
  homePage = 'https://github.com/nf-core/sicilian'
  description = 'Run SICILIAN alternative splicing analysis pipeline for single-cell RNA-seq data'
  mainScript = 'main.nf'
  nextflowVersion = '>=21.04.0'
  version = '1.0dev'
}

The base.config is as follows:

process {

  // TODO nf-core: Check the defaults for all processes
  cpus = { check_max( 1 * task.attempt, 'cpus' ) }
  memory = { check_max( 7.GB * task.attempt, 'memory' ) }
  time = { check_max( 4.h * task.attempt, 'time' ) }

  errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' }
  maxRetries = 1
  maxErrors = '-1'

  // Process-specific resource requirements
  // NOTE - Only one of the labels below are used in the fastqc process in the main script.
  //        If possible, it would be nice to keep the same label naming convention when
  //        adding in your processes.
  // TODO nf-core: Customise requirements for specific processes.
  // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors
  withLabel:process_low {
    cpus = { check_max( 2 * task.attempt, 'cpus' ) }
    memory = { check_max( 14.GB * task.attempt, 'memory' ) }
    time = { check_max( 6.h * task.attempt, 'time' ) }
  }
  withLabel:process_medium {
    cpus = { check_max( 6 * task.attempt, 'cpus' ) }
    memory = { check_max( 42.GB * task.attempt, 'memory' ) }
    time = { check_max( 8.h * task.attempt, 'time' ) }
  }
  withLabel:process_high {
    cpus = { check_max( 12 * task.attempt, 'cpus' ) }
    memory = { check_max( 84.GB * task.attempt, 'memory' ) }
    time = { check_max( 10.h * task.attempt, 'time' ) }
  }
  withLabel:process_higher {
    cpus = 40
    memory = 1024.GB
    time = 8.h
  }
  withLabel:process_long {
    time = { check_max( 20.h * task.attempt, 'time' ) }
  }
  withLabel:cpu_2 {
    cpus = { check_max( 2 * task.attempt, 'cpus' ) }
  }
  withLabel:process_super_highmem {
    memory = { check_max( 200.GB * task.attempt, 'memory' ) }
  }
  withLabel:process_super_long {
    time = { check_max( 48.h * task.attempt, 'time' ) }
  }
}

I create a new label process_higher to hard code the parameters of cpu and memory. However, when I execute the tasks, no matter how many CPUs I set, it only uses one core to run.

The following file is my nextflow.log

nextflow.log

[bentsherman]

It would be good to isolate the issue in a small example, if you think it is a bug