Hi there QIIME2 community,
I'm pretty new to QIIME 2 but ran through a couple of tutorials without any issues. Now that I am trying to run my own data I keep encountering one issue with dada2, and reading through other topics and trying those solutions has only changed the error code that dada2 has been returning. It seems for now I keep getting stuck on the following:
> qiime dada2 denoise-paired \
> --i-demultiplexed-seqs demux-paired-end.qza \
> --p-trim-left-f 20 \
> --p-trim-left-r 20 \
> --p-trunc-len-f 270 \
> --p-trunc-len-r 270 \
> --o-representative-sequences rep-seqs.qza \
> --o-table unfiltered-table.qza \
> --o-denoising-stats stats.qza
> Plugin error from dada2:
>
> An error was encountered while running DADA2 in R (return code -11), please inspect stdout and stderr to learn more.
>
> Debug info has been saved to /tmp/qiime2-q2cli-err-41ramwjb.log
The following information is in the error log:
> Running external command line application(s). This may print messages to stdout and/or stderr.
> The command(s) being run are below. These commands cannot be manually re-run as they will depend on temporary files that no longer exist.
>
> Command: run_dada_paired.R /tmp/tmpa5etkafu/forward /tmp/tmpa5etkafu/reverse /tmp/tmpa5etkafu/output.tsv.biom /tmp/tmpa5etkafu/track.tsv /tmp/tmpa5etkafu/filt_f /tmp/tmpa5etkafu/filt_r 270 270 20 20 2.0 2 consensus 1.0 1 1000000
>
> R version 3.4.1 (2017-06-30)
> Loading required package: Rcpp
> DADA2 R package version: 1.6.0
> 1) Filtering
> *** caught segfault ***
> address (nil), cause 'unknown'
>
> Traceback:
> 1: .Call("_dada2_C_matchRef", PACKAGE = "dada2", seqs, ref, word_size, non_overlapping)
> 2: C_matchRef(seqs, sq.phix, wordSize, nonOverlapping)
> 3: isPhiX(as(sread(fqR), "character"), ...)
> 4: (function (fn, fout, maxN = c(0, 0), truncQ = c(2, 2), truncLen = c(0, 0), maxLen = c(Inf, Inf), minLen = c(20, 20), trimLeft = c(0, 0), minQ = c(0, 0), maxEE = c(Inf, Inf), rm.phix = c(TRUE, TRUE), matchIDs = FALSE, primer.fwd = NULL, id.sep = "\\s", id.field = NULL, n = 1e+06, OMP = TRUE, compress = TRUE, verbose = FALSE, ...) { if (!OMP) { ompthreads <- .Call(ShortRead:::.set_omp_threads, 1L) on.exit(.Call(ShortRead:::.set_omp_threads, ompthreads)) } if (!is.character(fn) || length(fn) != 2) stop("Two paired input file names required.") if (!is.character(fout) || length(fout) != 2) stop("Two paired output file names required.") if (any(duplicated(c(fn, fout)))) { stop("The output and input file names must be different.") } for (var in c("maxN", "truncQ", "truncLen", "maxLen", "minLen", "trimLeft", "minQ", "maxEE", "rm.phix")) { if (length(get(var)) == 1) { assign(var, c(get(var), get(var))) } if (length(get(var)) != 2) { stop(paste("Input variable", var, "must be length 1 or 2 (Forward, Reverse).")) } } startF <- max(1, trimLeft[[1]] + 1, na.rm = TRUE) startR <- max(1, trimLeft[[2]] + 1, na.rm = TRUE) endF <- truncLen[[1]] if (endF < startF) { endF = NA } endF <- endF - startF + 1 endR <- truncLen[[2]] if (endR < startR) { endR = NA } endR <- endR - startR + 1 fF <- FastqStreamer(fn[[1]], n = n) on.exit(close(fF)) fR <- FastqStreamer(fn[[2]], n = n) on.exit(close(fR), add = TRUE) if (file.exists(fout[[1]])) { if (file.remove(fout[[1]])) { if (verbose) message("Overwriting file:", fout[[1]]) } else { stop("Failed to overwrite file:", fout[[1]]) } } if (file.exists(fout[[2]])) { if (file.remove(fout[[2]])) { if (verbose) message("Overwriting file:", fout[[2]]) } else { stop("Failed to overwrite file:", fout[[2]]) } } first = TRUE remainderF <- ShortReadQ() remainderR <- ShortReadQ() casava <- "Undetermined" inseqs = 0 outseqs = 0 while (TRUE) { suppressWarnings(fqF <- yield(fF)) suppressWarnings(fqR <- yield(fR)) if (length(fqF) == 0 && length(fqR) == 0) { break } inseqs <- inseqs + length(fqF) if (matchIDs) { if (first) { if (is.null(id.field)) { id1 <- as.character(id(fqF)[[1]]) id.fields <- strsplit(id1, id.sep)[[1]] ncolon <- sapply(gregexpr(":", id.fields), length) ncoltab <- table(ncolon) if (max(ncolon) == 6 && ncoltab["6"] == 1) { casava <- "Current" id.field <- which(ncolon == 6) } else if (max(ncolon) == 4 && ncoltab["4"] == 1) { casava <- "Old" id.field <- which(ncolon == 4) } else { stop("Couldn't automatically detect the sequence identifier field in the fastq id string.") } } } else { fqF <- append(remainderF, fqF) fqR <- append(remainderR, fqR) } } else { if (length(fqF) != length(fqR)) stop("Mismatched forward and reverse sequence files: ", length(fqF), ", ", length(fqR), ".") } if (matchIDs) { idsF <- sapply(strsplit(as.character(id(fqF)), id.sep), `[`, id.field) idsR <- sapply(strsplit(as.character(id(fqR)), id.sep), `[`, id.field) if (casava == "Old") { idsF <- sapply(strsplit(idsF, "#"), `[`, 1) } lastF <- max(c(0, which(idsF %in% idsR))) lastR <- max(c(0, which(idsR %in% idsF))) if (lastF < length(fqF)) { remainderF <- fqF[(lastF + 1):length(fqF)] } else { remainderF <- ShortReadQ() } if (lastR < length(fqR)) { remainderR <- fqR[(lastR + 1):length(fqR)] } else { remainderR <- ShortReadQ() } fqF <- fqF[idsF %in% idsR] fqR <- fqR[idsR %in% idsF] } if (!is.null(primer.fwd)) { barlen <- nchar(primer.fwd) keepF <- narrow(sread(fqF), 1, barlen) == primer.fwd keepR <- (narrow(sread(fqR), 1, barlen) == primer.fwd) & !keepF fq <- ShortReadQ(sread = c(sread(fqF[keepF]), sread(fqR[keepR])), quality = c(quality(quality(fqF[keepF])), quality(quality(fqR[keepR]))), id = c(id(fqF[keepF]), id(fqR[keepR]))) fqR <- ShortReadQ(sread = c(sread(fqR[keepF]), sread(fqF[keepR])), quality = c(quality(quality(fqR[keepF])), quality(quality(fqF[keepR]))), id = c(id(fqR[keepF]), id(fqF[keepR]))) fqF <- fq rm(fq) } if (is.finite(maxLen[[1]]) || is.finite(maxLen[[2]])) { keep <- width(fqF) <= maxLen[[1]] & width(fqR) <= maxLen[[2]] fqF <- fqF[keep] fqR <- fqR[keep] } keep <- (width(fqF) >= startF & width(fqR) >= startR) fqF <- fqF[keep] fqF <- narrow(fqF, start = startF, end = NA) fqR <- fqR[keep] fqR <- narrow(fqR, start = startR, end = NA) encF <- encoding(quality(fqF)) encR <- encoding(quality(fqR)) if (is.numeric(truncQ)) { indF <- which(encF == truncQ[[1]]) indR <- which(encR == truncQ[[2]]) if (!(length(indF) == 1 && length(indR) == 1)) stop("Encoding for this truncQ value not found.") truncQ <- c(names(encF)[[indF]], names(encR)[[indR]]) } if (length(fqF) > 0) { rngF <- trimTails(fqF, 1, truncQ[[1]], ranges = TRUE) fqF <- narrow(fqF, 1, end(rngF)) } if (length(fqR) > 0) { rngR <- trimTails(fqR, 1, truncQ[[2]], ranges = TRUE) fqR <- narrow(fqR, 1, end(rngR)) } truncQ <- c(encF[truncQ[1]], encR[truncQ[2]]) keep <- (width(fqF) > 0 & width(fqR) > 0) fqF <- fqF[keep] fqR <- fqR[keep] keep <- rep(TRUE, length(fqF)) if (!is.na(endF)) { keep <- keep & (width(fqF) >= endF) } if (!is.na(endR)) { keep <- keep & (width(fqR) >= endR) } fqF <- fqF[keep] fqR <- fqR[keep] fqF <- narrow(fqF, start = 1, end = endF) fqR <- narrow(fqR, start = 1, end = endR) keep <- width(fqF) >= minLen[[1]] & width(fqR) >= minLen[[2]] fqF <- fqF[keep] fqR <- fqR[keep] suppressWarnings(keep <- nFilter(maxN[[1]])(fqF) & nFilter(maxN[[2]])(fqR)) fqF <- fqF[keep] fqR <- fqR[keep] keep <- rep(TRUE, length(fqF)) qmat <- as(quality(fqF), "matrix") if (minQ[[1]] > truncQ[[1]]) suppressWarnings(keep <- keep & (apply(qmat, 1, min) > minQ[[1]])) if (maxEE[[1]] < Inf) keep <- keep & C_matrixEE(qmat) <= maxEE[[1]] qmat <- as(quality(fqR), "matrix") if (minQ[[2]] > truncQ[[2]]) suppressWarnings(keep <- keep & (apply(qmat, 1, min) > minQ[[2]])) if (maxEE[[2]] < Inf) keep <- keep & C_matrixEE(qmat) <= maxEE[[2]] fqF <- fqF[keep] fqR <- fqR[keep] rm(qmat) if (length(fqF) != length(fqR)) stop("Filtering caused mismatch between forward and reverse sequence lists: ", length(fqF), ", ", length(fqR), ".") if (rm.phix[[1]] && rm.phix[[2]]) { is.phi <- isPhiX(as(sread(fqF), "character"), ...) is.phi <- is.phi | isPhiX(as(sread(fqR), "character"), ...) } else if (rm.phix[[1]] && !rm.phix[[2]]) { is.phi <- isPhiX(as(sread(fqF), "character"), ...) } else if (!rm.phix[[1]] && rm.phix[[2]]) { is.phi <- isPhiX(as(sread(fqR), "character"), ...) } if (any(rm.phix)) { fqF <- fqF[!is.phi] fqR <- fqR[!is.phi] } outseqs <- outseqs + length(fqF) if (first) { writeFastq(fqF, fout[[1]], "w", compress = compress) writeFastq(fqR, fout[[2]], "w", compress = compress) first = FALSE } else { writeFastq(fqF, fout[[1]], "a", compress = compress) writeFastq(fqR, fout[[2]], "a", compress = compress) } } if (outseqs == 0) { } if (verbose) { outperc <- round(outseqs * 100/inseqs, 1) outperc <- paste(" (", outperc, "%)", sep = "") message("Read in ", inseqs, " paired-sequences, output ", outseqs, outperc, " filtered paired-sequences.", sep = "") } if (outseqs == 0) { message(paste("The filter removed all reads:", fout[[1]], "and", fout[[2]], "not written.")) file.remove(fout[[1]]) file.remove(fout[[2]]) } return(invisible(c(reads.in = inseqs, reads.out = outseqs)))})(dots[[1L]][[1L]], dots[[2L]][[1L]], truncQ = 2L, truncLen = c(270L, 270L), trimLeft = c(20L, 20L), maxLen = Inf, minLen = 20, maxN = 0, minQ = 0, maxEE = 2, rm.phix = TRUE, primer.fwd = NULL, matchIDs = FALSE, id.sep = "\\s", id.field = NULL, n = 1e+05, OMP = TRUE, compress = TRUE, verbose = FALSE)
> 5: .mapply(FUN, dots, MoreArgs)
> 6: FUN(X[[i]], ...)
> 7: lapply(X = X, FUN = FUN, ...)
> 8: mclapply(seq_len(n), do_one, mc.preschedule = mc.preschedule, mc.set.seed = mc.set.seed, mc.silent = mc.silent, mc.cores = mc.cores, mc.cleanup = mc.cleanup)
> 9: mcmapply(fastqPairedFilter, mapply(c, fwd, rev, SIMPLIFY = FALSE), mapply(c, filt, filt.rev, SIMPLIFY = FALSE), MoreArgs = list(truncQ = truncQ, truncLen = truncLen, trimLeft = trimLeft, maxLen = maxLen, minLen = minLen, maxN = maxN, minQ = minQ, maxEE = maxEE, rm.phix = rm.phix, primer.fwd = primer.fwd, matchIDs = matchIDs, id.sep = id.sep, id.field = id.field, n = n, OMP = OMP, compress = compress, verbose = verbose), mc.cores = ncores, mc.silent = TRUE)
> 10: filterAndTrim(unfiltsF, filtsF, unfiltsR, filtsR, truncLen = c(truncLenF, truncLenR), trimLeft = c(trimLeftF, trimLeftR), maxEE = maxEE, truncQ = truncQ, rm.phix = TRUE, multithread = multithread)
> 11: withCallingHandlers(expr, warning = function(w) invokeRestart("muffleWarning"))
> 12: suppressWarnings(filterAndTrim(unfiltsF, filtsF, unfiltsR, filtsR, truncLen = c(truncLenF, truncLenR), trimLeft = c(trimLeftF, trimLeftR), maxEE = maxEE, truncQ = truncQ, rm.phix = TRUE, multithread = multithread))
> An irrecoverable exception occurred. R is aborting now ...
> Traceback (most recent call last):
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/site-packages/q2_dada2/_denoise.py", line 229, in denoise_paired
> run_commands([cmd])
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/site-packages/q2_dada2/_denoise.py", line 36, in run_commands
> subprocess.run(cmd, check=True)
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/subprocess.py", line 398, in run
> output=stdout, stderr=stderr)
> subprocess.CalledProcessError: Command '['run_dada_paired.R', '/tmp/tmpa5etkafu/forward', '/tmp/tmpa5etkafu/reverse', '/tmp/tmpa5etkafu/output.tsv.biom', '/tmp/tmpa5etkafu/track.tsv', '/tmp/tmpa5etkafu/filt_f', '/tmp/tmpa5etkafu/filt_r', '270', '270', '20', '20', '2.0', '2', 'consensus', '1.0', '1', '1000000']' returned non-zero exit status -11
>
> During handling of the above exception, another exception occurred:
>
> Traceback (most recent call last):
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/site-packages/q2cli/commands.py", line 274, in __call__
> results = action(**arguments)
> File "<decorator-gen-380>", line 2, in denoise_paired
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/site-packages/qiime2/sdk/action.py", line 232, in bound_callable
> output_types, provenance)
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/site-packages/qiime2/sdk/action.py", line 367, in _callable_executor_
> output_views = self._callable(**view_args)
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/site-packages/q2_dada2/_denoise.py", line 244, in denoise_paired
> " and stderr to learn more." % e.returncode)
> Exception: An error was encountered while running DADA2 in R (return code -11), please inspect stdout and stderr to learn more.
The data I am trying to analyse is comprised of many paired fastq files, which I have imported using:
> qiime tools import --type 'SampleData[PairedEndSequencesWithQuality]' --input-path raw/ --source-format CasavaOneEightSingleLanePerSampleDirFmt --output-path demux-paired-end.qza
Is there anybody that can help me out here? I have tried the following solutions from other topics:
- echo ".libPaths(.libPaths()[2])" > $HOME/.Rprofile
- CDPATH= R -e 'source("https://bioconductor.org/biocLite.R"); biocLite("dada2")'
- conda create -n DONOTUSE --file https://data.qiime2.org/distro/core/qiime2-2017.12-conda-osx-64.txt
source activate DONOTUSE
wget -qO- https:/uploads/qiime21/original/1X/ac621debfd04f561bf1075b4efa56c0e947a1928.txt > which run_dada_single.R
wget -qO- https:/uploads/qiime21/original/1X/07fa5193f904022996a35a963b12b9ac79dbb1f4.txt 2 > which run_dada_paired.R
The last command couldn't run all the way till the end, as a lot of the URLs returned "404 not found". The command is for an older version of QIIME 2, so I thought maybe that's why? Anyway, since it did overwrite a bunch of stuff, I tried my command again and I got a little bit further, and my computer ran for 2 days straight until I came home to the bright red error message once again (grr!). Here is the latest error log file for this:
> Running external command line application(s). This may print messages to stdout and/or stderr.
> The command(s) being run are below. These commands cannot be manually re-run as they will depend on temporary files that no longer exist.
>
> Command: run_dada_paired.R /tmp/tmpyy5l777i/forward /tmp/tmpyy5l777i/reverse /tmp/tmpyy5l777i/output.tsv.biom /tmp/tmpyy5l777i/track.tsv /tmp/tmpyy5l777i/filt_f /tmp/tmpyy5l777i/filt_r 270 270 20 20 2.0 2 consensus 1.0 1 1000000
>
> R version 3.4.1 (2017-06-30)
> Loading required package: Rcpp
> DADA2 R package version: 1.6.0
> 1) Filtering ..................................................
> 2) Learning Error Rates
> 2a) Forward Reads
> Initializing error rates to maximum possible estimate.
> Sample 1 - 7601 reads in 5793 unique sequences.
> Sample 2 - 4809 reads in 3892 unique sequences.
> Sample 3 - 7806 reads in 5801 unique sequences.
> Sample 4 - 6017 reads in 4632 unique sequences.
> Sample 5 - 3728 reads in 3191 unique sequences.
> Sample 6 - 4592 reads in 3711 unique sequences.
> Sample 7 - 4100 reads in 3613 unique sequences.
> Sample 8 - 4574 reads in 3527 unique sequences.
> Sample 9 - 6076 reads in 3304 unique sequences.
> Sample 10 - 4625 reads in 3655 unique sequences.
> Sample 11 - 4649 reads in 3541 unique sequences.
> Sample 12 - 9846 reads in 7379 unique sequences.
> Sample 13 - 10766 reads in 8500 unique sequences.
> Sample 14 - 5510 reads in 4475 unique sequences.
> Sample 15 - 8833 reads in 6268 unique sequences.
> Sample 16 - 7700 reads in 5483 unique sequences.
> Sample 17 - 4720 reads in 3405 unique sequences.
> Sample 18 - 48073 reads in 23165 unique sequences.
> Sample 19 - 69307 reads in 34940 unique sequences.
> Sample 20 - 35694 reads in 21669 unique sequences.
> Sample 21 - 57885 reads in 25015 unique sequences.
> Sample 22 - 63690 reads in 26008 unique sequences.
> Sample 23 - 70913 reads in 32543 unique sequences.
> Sample 24 - 71703 reads in 32864 unique sequences.
> Sample 25 - 41274 reads in 18804 unique sequences.
> Sample 26 - 81075 reads in 35381 unique sequences.
> Sample 27 - 59062 reads in 28023 unique sequences.
> Sample 28 - 69213 reads in 28648 unique sequences.
> Sample 29 - 65479 reads in 29352 unique sequences.
> Sample 30 - 1639 reads in 1487 unique sequences.
> Sample 31 - 1446 reads in 1290 unique sequences.
> Sample 32 - 1487 reads in 1405 unique sequences.
> Sample 33 - 1122 reads in 1070 unique sequences.
> Sample 34 - 1508 reads in 1340 unique sequences.
> Sample 35 - 1037 reads in 889 unique sequences.
> Sample 36 - 1233 reads in 1179 unique sequences.
> Sample 37 - 646 reads in 606 unique sequences.
> Sample 38 - 834 reads in 795 unique sequences.
> Sample 39 - 1506 reads in 1379 unique sequences.
> Sample 40 - 2446 reads in 2169 unique sequences.
> Sample 41 - 4450 reads in 3768 unique sequences.
> Sample 42 - 3324 reads in 3005 unique sequences.
> Sample 43 - 5128 reads in 4339 unique sequences.
> Sample 44 - 4228 reads in 3685 unique sequences.
> Sample 45 - 3426 reads in 2935 unique sequences.
> Sample 46 - 4717 reads in 3891 unique sequences.
> Sample 47 - 21089 reads in 14368 unique sequences.
> Sample 48 - 11007 reads in 9181 unique sequences.
> Sample 49 - 3811 reads in 3384 unique sequences.
> Sample 50 - 5319 reads in 4818 unique sequences.
> selfConsist step 2
> selfConsist step 3
> selfConsist step 4
> selfConsist step 5
> selfConsist step 6
> selfConsist step 7
> selfConsist step 8
>
> *** caught segfault ***
> address (nil), cause 'unknown'
>
> Traceback:
> 1: .Call("_dada2_dada_uniques", PACKAGE = "dada2", seqs, abundances, err, quals, score, gap, use_kmers, kdist_cutoff, band_size, omegaA, max_clust, min_fold, min_hamming, min_abund, use_quals, final_consensus, vectorized_alignment, homo_gap, multithread, verbose, SSE)
> 2: dada_uniques(names(derep[[i]]$uniques), unname(derep[[i]]$uniques), err, qi, opts[["SCORE_MATRIX"]], opts[["GAP_PENALTY"]], opts[["USE_KMERS"]], opts[["KDIST_CUTOFF"]], opts[["BAND_SIZE"]], opts[["OMEGA_A"]], if (initializeErr) { 1 } else { opts[["MAX_CLUST"]] }, opts[["MIN_FOLD"]], opts[["MIN_HAMMING"]], opts[["MIN_ABUNDANCE"]], TRUE, FALSE, opts[["VECTORIZED_ALIGNMENT"]], opts[["HOMOPOLYMER_GAP_PENALTY"]], multithread, (verbose >= 2), opts[["SSE"]])
> 3: dada(drpsF, err = NULL, selfConsist = TRUE, multithread = multithread, VECTORIZED_ALIGNMENT = FALSE, SSE = 1)
> An irrecoverable exception occurred. R is aborting now ...
> Traceback (most recent call last):
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/site-packages/q2_dada2/_denoise.py", line 229, in denoise_paired
> run_commands([cmd])
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/site-packages/q2_dada2/_denoise.py", line 36, in run_commands
> subprocess.run(cmd, check=True)
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/subprocess.py", line 398, in run
> output=stdout, stderr=stderr)
> subprocess.CalledProcessError: Command '['run_dada_paired.R', '/tmp/tmpyy5l777i/forward', '/tmp/tmpyy5l777i/reverse', '/tmp/tmpyy5l777i/output.tsv.biom', '/tmp/tmpyy5l777i/track.tsv', '/tmp/tmpyy5l777i/filt_f', '/tmp/tmpyy5l777i/filt_r', '270', '270', '20', '20', '2.0', '2', 'consensus', '1.0', '1', '1000000']' returned non-zero exit status -11
>
> During handling of the above exception, another exception occurred:
>
> Traceback (most recent call last):
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/site-packages/q2cli/commands.py", line 274, in __call__
> results = action(**arguments)
> File "<decorator-gen-380>", line 2, in denoise_paired
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/site-packages/qiime2/sdk/action.py", line 232, in bound_callable
> output_types, provenance)
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/site-packages/qiime2/sdk/action.py", line 367, in _callable_executor_
> output_views = self._callable(**view_args)
> File "/home/qiime2/miniconda/envs/qiime2-2018.6/lib/python3.5/site-packages/q2_dada2/_denoise.py", line 244, in denoise_paired
> " and stderr to learn more." % e.returncode)
> Exception: An error was encountered while running DADA2 in R (return code -11), please inspect stdout and stderr to learn more.
Sorry, that's a lot of error log files all at once, but I'm hoping someone can make some sense out of it.
I'd appreciate any help!
Cheers,
Bobby
