package jgi;

import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collections;

import dna.AminoAcid;
import fileIO.ByteFile;
import fileIO.ByteStreamWriter;
import fileIO.FileFormat;
import fileIO.ReadWrite;
import jgi.Repeat.PosComparator2;
import kmer.AbstractKmerTableSet;
import kmer.KmerTableSet;
import shared.Parse;
import shared.Parser;
import shared.PreParser;
import shared.ReadStats;
import shared.Shared;
import shared.Timer;
import shared.Tools;
import stream.ConcurrentReadInputStream;
import stream.ConcurrentReadOutputStream;
import stream.FASTQ;
import stream.FastaReadInputStream;
import stream.Read;
import structures.ByteBuilder;
import structures.CRange;
import structures.EntropyTracker;
import structures.ListNum;
import template.Accumulator;
import template.ThreadWaiter;

/**
 * This class finds probable repeats using kmer-matching,
 * and prints them out ordered by depth, then length.
 * 
 * The repeats are only "probable" because no alignment is done;
 * a sequence is considered a probable repeat of depth D if
 * if all kmers within it have a depth of at least D.
 * 
 * @author Brian Bushnell
 * @date June 15, 2023
 *
 */
public class RepeatFinder implements Accumulator<RepeatFinder.ProcessThread> {
	
	/*--------------------------------------------------------------*/
	/*----------------        Initialization        ----------------*/
	/*--------------------------------------------------------------*/
	
	/**
	 * Code entrance from the command line.
	 * @param args Command line arguments
	 */
	public static void main(String[] args){
		//Start a timer immediately upon code entrance.
		Timer t=new Timer();
		
		//Create an instance of this class
		RepeatFinder x=new RepeatFinder(args);
		
		//Run the object
		x.process(t);
		
		//Close the print stream if it was redirected
		Shared.closeStream(x.outstream);
	}
	
	/**
	 * Constructor.
	 * @param args Command line arguments
	 */
	public RepeatFinder(String[] args){
		
		{//Preparse block for help, config files, and outstream
			PreParser pp=new PreParser(args, getClass(), false);
			args=pp.args;
			outstream=pp.outstream;
		}
		
		//Set shared static variables prior to parsing
		ReadWrite.USE_PIGZ=ReadWrite.USE_UNPIGZ=true;
		ReadWrite.MAX_ZIP_THREADS=Shared.threads();
		FASTQ.TEST_INTERLEAVED=FASTQ.FORCE_INTERLEAVED=false;
		
		{//Parse the arguments
			final Parser parser=parse(args);
			Parser.processQuality();
			
			maxReads=parser.maxReads;
			overwrite=ReadStats.overwrite=parser.overwrite;
			append=ReadStats.append=parser.append;
			
			in1=parser.in1;
			extin=parser.extin;

			outr=parser.out1;
			extout=parser.extout;
			
			k=Repeat.K=parser.k;
			assert(k>=1 && k<=32) : "1<=k<=32: k="+k;
			amino=Shared.AMINO_IN;
		}

		validateParams();
		fixExtensions(); //Add or remove .gz or .bz2 as needed
		checkFileExistence(); //Ensure files can be read and written
		checkStatics(); //Adjust file-related static fields as needed for this program 
		
		//Create output FileFormat objects
		ffoutr=FileFormat.testOutput(outr, FileFormat.TXT, null, true, overwrite, append, false);
		ffouts=FileFormat.testOutput(outs, FileFormat.FA, extout, true, overwrite, append, ordered);

		//Create input FileFormat objects
		ffin1=FileFormat.testInput(in1, FileFormat.FASTA, extin, true, true);
		
		tables=new KmerTableSet(args, 12);
		
		{//set some constants
			bitsPerBase=(amino ? 5 : 2);
			maxSymbol=(amino ? 20 : 3);
			symbols=maxSymbol+1;
			symbolArrayLen=(64+bitsPerBase-1)/bitsPerBase;
			symbolSpace=(1<<bitsPerBase);
			symbolMask=symbolSpace-1;
			
			symbolToNumber=AminoAcid.symbolToNumber(amino);
			symbolToNumber0=AminoAcid.symbolToNumber0(amino);
			symbolToComplementNumber0=AminoAcid.symbolToComplementNumber0(amino);
			
			clearMasks=new long[symbolArrayLen];
			leftMasks=new long[symbolArrayLen];
			rightMasks=new long[symbolArrayLen];
			setMasks=new long[symbols][symbolArrayLen];
			for(int i=0; i<symbolArrayLen; i++){
				clearMasks[i]=~(symbolMask<<(bitsPerBase*i));
				leftMasks[i]=((-1L)<<(bitsPerBase*i));
				rightMasks[i]=~((-1L)<<(bitsPerBase*i));
				for(long j=0; j<symbols; j++){
					setMasks[(int)j][i]=(j<<(bitsPerBase*i));
				}
			}
			
//			minlen=k-1;
			shift=bitsPerBase*k;
			shift2=shift-bitsPerBase;
			mask=(shift>63 ? -1L : ~((-1L)<<shift));
		}
		Repeat.initialize(maxGap, minRepeat, k);
		middleMask=(((k|1)==1) ? ~(3<<(k/2)) : ~(15<<(k/2-1)));//Works for even kmer lengths.
		
		baseMaskArray=new byte[127];
		for(int i=0; i<baseMaskArray.length; i++) {
			final byte b;
			if(!maskRepeats) {b=(byte)i;}
			else if(softMask) {b=Tools.toLowerCase((byte)i);
			}else{b=maskSymbol;}
			baseMaskArray[i]=b;
		}
	}
	
	/*--------------------------------------------------------------*/
	/*----------------    Initialization Helpers    ----------------*/
	/*--------------------------------------------------------------*/
	
	/** Parse arguments from the command line */
	private Parser parse(String[] args){
		
		//Create a parser object
		Parser parser=new Parser();
		
		//Set any necessary Parser defaults here
		parser.out1=outr;
		
		//Parse each argument
		for(int i=0; i<args.length; i++){
			String arg=args[i];
			
			//Break arguments into their constituent parts, in the form of "a=b"
			String[] split=arg.split("=");
			String a=split[0].toLowerCase();
			String b=split.length>1 ? split[1] : null;
			if(b!=null && b.equalsIgnoreCase("null")){b=null;}
			
			if(a.equals("verbose")){
				verbose=Parse.parseBoolean(b);
			}else if(a.equals("ordered")){
				ordered=Parse.parseBoolean(b);
			}else if(a.equals("displayprogress")){
				DISPLAY_PROGRESS=Parse.parseBoolean(b);
			}else if(a.equals("qhdist") || a.equals("hdist")){
				qHammingDist=Integer.parseInt(b);
			}else if(a.equals("maskmiddle")){
				maskMiddle=AbstractKmerTableSet.MASK_MIDDLE=Parse.parseBoolean(b);
			}else if(a.equals("minrepeat") || a.equals("minlen") || a.equals("minlength")){
				minRepeat=Integer.parseInt(b);
			}else if(a.equals("maxgap") || a.equals("gap")){
				maxGap=Integer.parseInt(b);
			}else if(a.equals("maxdepth") || a.equals("maxcount")){
				maxDepth=Integer.parseInt(b);
				if(maxDepth<1) {maxDepth=Shared.MAX_ARRAY_LEN;}
			}else if(a.equals("mindepth") || a.equals("mincount")){
				minDepth=Tools.max(2, Integer.parseInt(b));
			}else if(a.equals("printlen") || a.equals("seqlen") || a.equals("println") || 
					a.equals("preview") || a.equals("printsequence") || a.equals("printseq")){
				if(b==null || Character.isLetter(b.charAt(0))){
					if(Parse.parseBoolean(b)) {
						Repeat.SEQ_AFFIX_LEN=12;
					}else{
						Repeat.SEQ_AFFIX_LEN=0;
					}
				}else{
					int x=Integer.parseInt(b);
					Repeat.SEQ_AFFIX_LEN=(x-3)/2;
				}
			}
			
			else if(a.equals("mask") || a.equals("maskrepeats")){
				if("lc".equalsIgnoreCase(b) || "lowercase".equalsIgnoreCase(b) ||
						"soft".equalsIgnoreCase(b)) {
					maskRepeats=true;
					softMask=true;
				}else if("hard".equalsIgnoreCase(b)) {
					maskRepeats=true;
					softMask=false;
				}else if("t".equalsIgnoreCase(b) || "f".equalsIgnoreCase(b) || b==null || b.length()>1) {
					maskRepeats=Parse.parseBoolean(b);
				}else{
					maskRepeats=true;
					softMask=false;
					maskSymbol=(byte)b.charAt(0);
				}
			}else if(a.equals("softmask") || a.equals("soft")){
				softMask=Parse.parseBoolean(b);
				if(softMask) {maskRepeats=true;}
			}else if(a.equals("hardmask") || a.equals("hard")){
				softMask=!Parse.parseBoolean(b);
				if(!softMask) {maskRepeats=true;}
			}else if(a.equals("symbol") || a.equals("masksymbol")){
				softMask=false;
				maskSymbol=(byte)b.charAt(0);
			}else if(a.equals("weak") || a.equals("weaksubsume") || a.equals("weaksubsumes") || a.equals("ignoregaps")){
				weakSubsumes=Parse.parseBoolean(b);
			}
			
			else if(a.equals("printrepeats") || a.equals("pr") || a.equals("print")){
				printRepeats=Parse.parseBoolean(b);
				maskRepeats=!printRepeats;
			}
			
			else if(a.equals("outs") || a.equals("outm") || a.equals("outmasked")){
				outs=b;
			}else if(a.equals("parse_flag_goes_here")){
				long fake_variable=Parse.parseKMG(b);
				//Set a variable here
			}else if(parser.parse(arg, a, b)){//Parse standard flags in the parser
				//do nothing
			}else if(parser.parseK(arg, a, b)){//Parse k
				//do nothing
			}else{
				outstream.println("Unknown parameter "+args[i]);
				assert(false) : "Unknown parameter "+args[i];
			}
		}
		
		return parser;
	}
	
	/** Add or remove .gz or .bz2 as needed */
	private void fixExtensions(){
		in1=Tools.fixExtension(in1);
	}
	
	/** Ensure files can be read and written */
	private void checkFileExistence(){
		//Ensure output files can be written
		if(!Tools.testOutputFiles(overwrite, append, false, outr, outs)){
			outstream.println((outr==null)+", "+outr+", "+outs);
			throw new RuntimeException("\n\noverwrite="+overwrite+"; Can't write to output file "+outr+"\n");
		}
		
		//Ensure input files can be read
		if(!Tools.testInputFiles(false, true, in1)){
			throw new RuntimeException("\nCan't read some input files.\n");  
		}
		
		//Ensure that no file was specified multiple times
		if(!Tools.testForDuplicateFiles(true, in1, outr, outs)){
			throw new RuntimeException("\nSome file names were specified multiple times.\n");
		}
	}
	
	/** Adjust file-related static fields as needed for this program */
	private static void checkStatics(){
		//Adjust the number of threads for input file reading
		if(!ByteFile.FORCE_MODE_BF1 && !ByteFile.FORCE_MODE_BF2 && Shared.threads()>2){
			ByteFile.FORCE_MODE_BF2=true;
		}
		
		assert(FastaReadInputStream.settingsOK());
	}
	
	/** Ensure parameter ranges are within bounds and required parameters are set */
	private boolean validateParams(){
//		assert(minfoo>0 && minfoo<=maxfoo) : minfoo+", "+maxfoo;
//		assert(false) : "TODO";
		assert(k>=1 && k<=32) : k;
		return true;
	}
	
	/*--------------------------------------------------------------*/
	/*----------------         Outer Methods        ----------------*/
	/*--------------------------------------------------------------*/

	/** Create read streams and process all data */
	void process(Timer t){
		
		//Turn off read validation in the input threads to increase speed
		final boolean vic=Read.VALIDATE_IN_CONSTRUCTOR;
		Read.VALIDATE_IN_CONSTRUCTOR=Shared.threads()<4;
		
		AbstractKmerTableSet.DISPLAY_STATS=false;
		
		/* Fill tables with kmers */
		tables.process(t);
		
		if(DISPLAY_PROGRESS){
			outstream.println("After loading:");
			Shared.printMemory();
			outstream.println();
		}
		
		//Create a read input stream
		final ConcurrentReadInputStream cris=makeCris();
		
		//Optionally create a read output stream
		final ConcurrentReadOutputStream ros=makeCros();
		
		//Reset counters
		readsProcessed=readsOut=0;
		basesProcessed=basesOut=0;
		
		//Process the reads in separate threads
		spawnThreads(cris, ros);
		
		if(verbose){outstream.println("Finished; closing streams.");}
		
		//Write anything that was accumulated by ReadStats
		errorState|=ReadStats.writeAll();
		//Close the read streams
		errorState|=ReadWrite.closeStreams(cris, ros);
		
		//Reset read validation
		Read.VALIDATE_IN_CONSTRUCTOR=vic;
		
		//Show the stuff the person involved cares about
		printRepeats(ffoutr, masterListOfRepeats);
		
		//Report timing and results
		t.stop();
		outstream.println(Tools.timeReadsBasesProcessed(t, readsProcessed, basesProcessed, 8));
		outstream.println(Tools.readsBasesOut(readsProcessed, basesProcessed, readsOut, basesOut, 8, false));
		outstream.println(Tools.number("Repeat Bases:       ", repeatBases, 8)+String.format(" \t%.2f%%", repeatBases*100.0/basesProcessed));
		if(maskRepeats && ros!=null) {
			outstream.println(Tools.number("Bases Masked:       ", basesMasked, 8)+String.format(" \t%.2f%%", basesMasked*100.0/basesProcessed));
		}
		
		//Throw an exception of there was an error in a thread
		if(errorState){
			throw new RuntimeException(getClass().getName()+" terminated in an error state; the output may be corrupt.");
		}
	}
	
	void printRepeats(FileFormat ff, ArrayList<Repeat> list) {
		Collections.sort(list);
		Collections.reverse(list);
		if(ff==null){return;}
		ByteStreamWriter bsw=new ByteStreamWriter(ff);
		bsw.start();
		bsw.println(Repeat.tsvHeader());
		ByteBuilder bb=new ByteBuilder();
		for(Repeat r : list) {
			r.appendTo(bb);
			bsw.println(bb);
			bb.clear();
		}
		bsw.poisonAndWait();
	}
	
//	boolean printSubsumes=false; //TODO: For testing
	int subsume(ArrayList<Repeat> list, boolean weak) {
		if(list.isEmpty()) {return 0;}
		list.sort(Repeat.PosComparator.comparator);
		
		int removed=0;
		Repeat current=list.get(0);
		for(int i=1; i<list.size(); i++) {
			Repeat r=list.get(i);
			assert(current!=r);
			if(current.subsumes(r, weak)) {
//				if(printSubsumes) {
//					System.err.println(current+"\n"+r);
//				}
				list.set(i, null);
				removed++;
			}else{
				current=r;
			}
		}
		if(removed>0){Tools.condenseStrict(list);}
		return removed;
	}
	
	private int removeFullyContained(final ArrayList<Repeat> repeats) {
		if(repeats.size()<2) {return 0;}
		repeats.sort(PosComparator2.comparator);
		Repeat current=repeats.get(0);
		int removed=0;
		for(int i=1; i<repeats.size(); i++) {
			final Repeat r=repeats.get(i);
			if(current.spans(r)){
				repeats.set(i, null);
				removed++;
			}else{
				assert(!current.overlaps(r) || maxGap<=k) : "\n"+current+"\n"+r;
				current=r;
			}
		}
		if(removed>0){
			Tools.condenseStrict(repeats);
		}
		return removed;
	}
	
	private ConcurrentReadInputStream makeCris(){
		ConcurrentReadInputStream cris=ConcurrentReadInputStream.getReadInputStream(maxReads, true, ffin1, null);
		cris.start(); //Start the stream
		if(verbose){outstream.println("Started cris");}
		boolean paired=cris.paired();
		if(!ffin1.samOrBam()){outstream.println("Input is being processed as "+(paired ? "paired" : "unpaired"));}
		return cris;
	}
	
	private ConcurrentReadOutputStream makeCros(){
		if(ffouts==null){return null;}

		//Select output buffer size based on whether it needs to be ordered
		final int buff=(ordered ? Tools.mid(16, 128, (Shared.threads()*2)/3) : 8);

		final ConcurrentReadOutputStream ros=ConcurrentReadOutputStream.getStream(ffouts, null, buff, null, false);
		ros.start(); //Start the stream
		return ros;
	}
	
	/*--------------------------------------------------------------*/
	/*----------------       Thread Management      ----------------*/
	/*--------------------------------------------------------------*/
	
	/** Spawn process threads */
	private void spawnThreads(final ConcurrentReadInputStream cris, final ConcurrentReadOutputStream ros){
		
		//Do anything necessary prior to processing
		
		//Determine how many threads may be used
		final int threads=Shared.threads();
		
		//Fill a list with ProcessThreads
		ArrayList<ProcessThread> alpt=new ArrayList<ProcessThread>(threads);
		for(int i=0; i<threads; i++){
			alpt.add(new ProcessThread(cris, ros, i));
		}
		
		//Start the threads and wait for them to finish
		boolean success=ThreadWaiter.startAndWait(alpt, this);
		errorState&=!success;
		
		//Do anything necessary after processing
		
	}
	
	@Override
	public final void accumulate(ProcessThread pt){
		synchronized(pt) {
			readsProcessed+=pt.readsProcessedT;
			basesProcessed+=pt.basesProcessedT;
			repeatBases+=pt.repeatBasesT;
			basesMasked+=pt.basesMaskedT;
			readsOut+=pt.readsOutT;
			basesOut+=pt.basesOutT;
			errorState|=(!pt.success);
			masterListOfRepeats.addAll(pt.oldRepeats);
		}
	}
	
	@Override
	public final boolean success(){return !errorState;}
	
	/*--------------------------------------------------------------*/
	/*----------------         Inner Methods        ----------------*/
	/*--------------------------------------------------------------*/
	
	final long rcomp(long kmer, int len){
		return amino ? kmer : AminoAcid.reverseComplementBinaryFast(kmer, len);
	}
	
	final int maskRepeat(Repeat r) {
		return maskRepeat(r.contig.bases, r.start, r.stop);
	}
	
	final int maskRepeat(CRange r) {
		return maskRepeat(((Read)r.o).bases, r.a, r.b);
	}
	
	final int maskRepeat(byte[] bases, int start, int stop) {
		int masked=0;
		for(int i=start; i<=stop; i++) {
			byte b=bases[i];
			byte b2=baseMaskArray[b];
			bases[i]=b2;
			masked+=(b2==b ? 0 : 1);
		}
		return masked;
	}
	
	/*--------------------------------------------------------------*/
	/*----------------         Inner Classes        ----------------*/
	/*--------------------------------------------------------------*/
	
	/** This class is static to prevent accidental writing to shared variables.
	 * It is safe to remove the static modifier. */
	class ProcessThread extends Thread {
		
		//Constructor
		ProcessThread(final ConcurrentReadInputStream cris_, final ConcurrentReadOutputStream ros_, final int tid_){
			cris=cris_;
			ros=ros_;
			tid=tid_;
		}
		
		//Called by start()
		@Override
		public void run(){
			//Do anything necessary prior to processing

			synchronized(this) {
				//Process the reads
				processInner();

				//Do anything necessary after processing
				for(Repeat r : openRepeats) {
					if(r.contigNum>=0 && r.depth>1 && r.length()>=minRepeat){
						assert(false);
						oldRepeats.add(r.clone());
					}
				}
				
//				printSubsumes=true;
				int subsumed=subsume(oldRepeats, weakSubsumes);
				assert(subsumed==0) : subsumed; //TODO:  Should be true since they are already removed, but test it.
//				for(Repeat r : oldRepeats) {
//					r.calcStats(et);
//				}

				//Indicate successful exit status
				success=true;
			}
		}
		
		/** Iterate through the reads */
		void processInner(){
			
			//Grab the first ListNum of reads
			ListNum<Read> ln=cris.nextList();
			
			//As long as there is a nonempty read list...
			while(ln!=null && ln.size()>0){
//				if(verbose){outstream.println("Fetched "+reads.size()+" reads.");} //Disabled due to non-static access
				
				processList(ln);
				
				//Notify the input stream that the list was used
				cris.returnList(ln);
//				if(verbose){outstream.println("Returned a list.");} //Disabled due to non-static access
				
				//Fetch a new list
				ln=cris.nextList();
			}
			
			//Notify the input stream that the final list was used
			if(ln!=null){
				cris.returnList(ln.id, ln.list==null || ln.list.isEmpty());
			}
		}
		
		void processList(ListNum<Read> ln){
			
			//Grab the actual read list from the ListNum
			final ArrayList<Read> reads=ln.list;
			
			int maxDepthSeen=0;
			assert(closedRepeats.isEmpty());
			
			//Loop through each read in the list
			for(int idx=0; idx<reads.size(); idx++){
				final Read r=reads.get(idx);
				
				//Validate reads in worker threads
				if(!r.validated()){r.validate(true);}

				//Track the initial length for statistics
				final int initialLength1=r.length();
				final int initialLength2=r.mateLength();

				//Increment counters
				readsProcessedT+=r.pairCount();
				basesProcessedT+=initialLength1+initialLength2;
				
				{
					//Reads are processed in this block.
					int depth=processRead(r);
					maxDepthSeen=Tools.max(maxDepthSeen, depth);
				}
			}
			
			collectResidual(maxDepthSeen);
			
			//This is occasionally needed due to lazy collection
			subsume(closedRepeats, weakSubsumes);
			
			ArrayList<CRange> ranges=toRanges(closedRepeats, true);
			
			//Output reads to the output stream
			if(ros!=null){
				ArrayList<Read> seqList=reads;
				if(maskRepeats) {
					if(!softMask && Repeat.SEQ_AFFIX_LEN>0) {makePreview(closedRepeats);}
					basesMaskedT+=maskRepeats(ranges);
				}else if(printRepeats) {seqList=fetchRepeatSequence(closedRepeats);}
				ros.add(seqList, ln.id);
				for(Read r : seqList) {
					readsOutT+=r.pairCount();
					basesOutT+=r.pairLength();
				}
			}
			repeatBasesT+=countRepeatBases(ranges);
			
			//Optionally clear sequence to save memory (but this interferes with printing repeat sequence)
//			for(Repeat r : closedRepeats) {r.contig=null;}
			
			oldRepeats.addAll(closedRepeats);
			closedRepeats.clear();
		}
		
		long maskRepeats(ArrayList<CRange> ranges) {
			long x=0;
			for(CRange r : ranges) {x+=maskRepeat(r);}
			return x;
		}
		
		long countRepeatBases(ArrayList<CRange> ranges) {
			long x=0;
			for(CRange r : ranges) {x+=r.length();}
			return x;
		}
		
		void makePreview(ArrayList<Repeat> list) {
			assert(maskRepeats && !softMask);
			if(list.isEmpty()){return;}
			ByteBuilder bb=new ByteBuilder(Repeat.SEQ_AFFIX_LEN*2+3);
			for(Repeat r : list) {r.setSeq(bb);}
		}
		
		void maskRepeats() {
			//Masking can be done on closedRepeats, but making a weakSubsumes list can save rework
			ArrayList<Repeat> list=closedRepeats;
			if(list.isEmpty()) {return;}
			if(!softMask) {
				//If hard masking, copies of the sequence need to be made first.
				//Skipping this will make the preview come out in lower case,
				//but it prevents memory crashes in the worst case.
				//Although it uses more memory in the expected case.
				ByteBuilder bb=new ByteBuilder(Repeat.SEQ_AFFIX_LEN*2+3);
				for(Repeat r : closedRepeats) {r.setSeq(bb);}
			}
			if(list.size()>1 && !weakSubsumes) {
				list=(ArrayList<Repeat>) closedRepeats.clone();
				subsume(list, true);
			}
			for(Repeat r : list) {
				maskRepeat(r);
			}
		}
		
		ArrayList<Read> fetchRepeatSequence(ArrayList<Repeat> repeats0){
			ArrayList<Read> reads=new ArrayList<Read>();
			if(repeats0.isEmpty()){return reads;}
			ArrayList<Repeat> repeats=(ArrayList<Repeat>) repeats0.clone();
			removeFullyContained(repeats);
			for(Repeat pete : repeats) {
				Read r=pete.toRead();
				reads.add(r);
			}
			return reads;
		}
		
		ArrayList<CRange> toRanges(ArrayList<Repeat> repeats, boolean merge){
			ArrayList<CRange> ranges=rangeBuffer;
			ranges.clear();
			if(repeats.size()==1) {ranges.add(repeats.get(0).toRange());}
			if(repeats.size()<2){return ranges;}
			{
				Repeat current=repeats.get(repeats.size()-1);
				for(int i=repeats.size()-2; i>=0; i--) {
					Repeat r=repeats.get(i);
					if(current.spans(r)) {
						//do nothing
					}else {
						ranges.add(current.toRange());
						current=r;
					}
				}
				ranges.add(current.toRange());
				Collections.sort(ranges);
			}
			if(merge && ranges.size()>1){
				CRange current=ranges.get(0);
				int removed=0;
				for(int i=1; i<ranges.size(); i++) {
					CRange r=ranges.get(i);
					if(current.touches(r)) {
						current.absorb(r);
						ranges.set(i, null);
						removed++;
					}else{
						current=r;
					}
				}
				if(removed>0){
					Tools.condenseStrict(ranges);
				}
			}
			return ranges;
		}
		
		/**
		 * Process a read.
		 * @param r1 Read 1
		 * @return max depth observed.
		 */
		int processRead(final Read r){
			if(r==null || r.length()<k) {return 0;}
			final byte[] bases=r.bases;
			
			long kmer=0;
			long rkmer=0;
			int len=0;
			
			final int start=0;
			final int stop=bases.length;
			int max=0;
			
			/* Loop through the bases, maintaining a forward and reverse kmer via bitshifts */
			for(int i=start; i<stop; i++){
				byte b=bases[i];
				long x=symbolToNumber0[b];
				long x2=symbolToComplementNumber0[b];
				kmer=((kmer<<bitsPerBase)|x)&mask;
				rkmer=((rkmer>>>bitsPerBase)|(x2<<shift2))&mask;
				if(forbidNs && !isFullyDefined(b)){len=0; rkmer=0;}else{len++;}
				//if(verbose){outstream.println("Scanning6b i="+i+", kmer="+kmer+", rkmer="+rkmer+", bases="+new String(bases, Tools.max(0, i-k2), Tools.min(i+1, k)));}
				if(len>=k){
					final int count=getValue(kmer, rkmer, qHammingDist);
					if(verbose){outstream.println("Testing kmer "+kmer+"; count="+count);}
					//assert(count>=1) : count;
					if(count>=minDepth) {
						increment(r, i, count);
						max=Tools.max(max, count);
					}
				}
			}
			
			return max;
		}
		
		/** TODO: This can go from O(N^2) to O(N) if it follows contours instead of incrementing all.
		 * Also, make sure min depth is implemented. */
		private void increment(Read contig, int pos, final int depth) {
			assert(pos>=k-1) : pos;
			int depth2=Tools.min(depth, maxDepth);
			while(depth2>=openRepeats.size()) {
				openRepeats.add(new Repeat(null, -2, openRepeats.size()));
			}
			for(int i=depth2; i>=minDepth; i--) {//Potentially quadratic
				Repeat r=openRepeats.get(i);
				Repeat old=r.increment(contig, pos, depth);
				if(old!=null){addRepeat(old);}
			}
		}
		
		private void collectResidual(int max) {
			max=Tools.min(max, openRepeats.size()-1); //In case maxDepth is set.
			assert(max<openRepeats.size()) : max+", "+openRepeats.size();
			for(int i=max; i>=minDepth; i--) {
				Repeat r=openRepeats.get(i);
				if(r.contigNum>=0 && r.depth>1 && r.length()>=minRepeat){
					addRepeat(r.clone());
					r.clear();
				}
			}
		}
		
		private void addRepeat(Repeat r){
			if(lastRepeat!=null && lastRepeat.subsumes(r, weakSubsumes)) {
				//for more efficiency I could prevent r from being created in the first place
				//but that would require adding complexity to Repeat
				return;
			}
			//Actually this can happen in rare cases due to lazy collection.
//			assert(lastRepeat==null || !r.subsumes(lastRepeat)) : "\n"+lastRepeat+"\n"+r;
			
			r.calcStats(et);
			closedRepeats.add(r);
			lastRepeat=r;
		}

		/**
		 * Transforms a kmer into all canonical values for a given Hamming distance.
		 * Returns the maximal count stored in the tables.
		 * @param kmer Forward kmer
		 * @param rkmer Reverse kmer
		 * @param qHDist Hamming distance
		 * @return Value stored in table, or -1
		 */
		private final int getValue(final long kmer, final long rkmer, final int qHDist){
			if(verbose){outstream.println("getValue()");}
			final int count=tables.getCount(kmer, rkmer);
			int maxCount=count;
			if(/*count<2 &&*/ qHDist>0){
				final int qHDist2=qHDist-1;

				//Sub
				for(int j=0; j<symbols/* && count<2*/; j++){
					for(int i=0; i<k /*&& id<1*/; i++){
						final long temp=(kmer&clearMasks[i])|setMasks[j][i];
						//					outstream.println(i+", "+j+", "+setMasks[j][i]+", "+qHDist);
						if(temp!=kmer){
							long rtemp=rcomp(temp, k);
							int keyCount=getValue(temp, rtemp, qHDist2);
							maxCount=Tools.max(keyCount, maxCount);
						}
					}
				}
			}
			return maxCount;
		}
		
		private Repeat lastRepeat=null;

		/** Number of reads processed by this thread */
		protected long readsProcessedT=0;
		/** Number of bases processed by this thread */
		protected long basesProcessedT=0;
		
		/** Number of reads retained by this thread */
		protected long readsOutT=0;
		/** Number of bases retained by this thread */
		protected long basesOutT=0;
		
		/** Number of repeat bases found by this thread */
		protected long repeatBasesT=0;
		
		/** Number of bases masked by this thread */
		protected long basesMaskedT=0;
		
		/** True only if this thread has completed successfully */
		boolean success=false;
		
		/** Shared input stream */
		private final ConcurrentReadInputStream cris;
		/** Shared output stream */
		private final ConcurrentReadOutputStream ros;
		/** Thread ID */
		final int tid;
		
		final EntropyTracker et=new EntropyTracker(4, Tools.min(50, Tools.max(k, minRepeat)), amino, 0, true);
		final ArrayList<Repeat> openRepeats=new ArrayList<Repeat>();
		final ArrayList<Repeat> closedRepeats=new ArrayList<Repeat>();
		final ArrayList<Repeat> oldRepeats=new ArrayList<Repeat>();
		final ArrayList<CRange> rangeBuffer=new ArrayList<CRange>();
	}
	
	/*--------------------------------------------------------------*/
	/*----------------            Fields            ----------------*/
	/*--------------------------------------------------------------*/

	/** Primary input file path */
	private String in1=null;

	/** Primary output file path */
	private String outr="stdout.txt";
	
	/** Sequence output */
	private String outs=null;
	
	/** Override input file extension */
	private String extin=null;
	/** Override output file extension */
	private String extout=null;
	
	/*--------------------------------------------------------------*/

	/** Number of reads processed */
	protected long readsProcessed=0;
	/** Number of bases processed */
	protected long basesProcessed=0;

	/** Number of reads retained */
	protected long readsOut=0;
	/** Number of bases retained */
	protected long basesOut=0;

	protected long repeatBases=0;
	protected long basesMasked=0;

	/** Quit after processing this many input reads; -1 means no limit */
	private long maxReads=-1;
	
	/** Display progress messages such as memory usage */
	public static boolean DISPLAY_PROGRESS=true;

	boolean forbidNs=true;

	boolean printRepeats;
	boolean maskRepeats=true;
	boolean softMask=true;
	byte maskSymbol='N';
	boolean weakSubsumes=false;
	
	int minRepeat=0;
	
	int maxGap=0;

	int maxDepth=Shared.MAX_ARRAY_LEN;
	int minDepth=2;
	
	/*--------------------------------------------------------------*/
	/*----------------         Final Fields         ----------------*/
	/*--------------------------------------------------------------*/

	/** Primary input file */
	private final FileFormat ffin1;
	
	/** Primary output file */
	private final FileFormat ffoutr;
	
	/** Sequence output file */
	private final FileFormat ffouts;
	
	private final int k;
	private final KmerTableSet tables;
	
	private final ArrayList<Repeat> masterListOfRepeats=new ArrayList<Repeat>();
	
	private final byte[] baseMaskArray;
	
	/*--------------------------------------------------------------*/
	/*-----------        Symbol-Specific Constants        ----------*/
	/*--------------------------------------------------------------*/

	/** True for amino acid data, false for nucleotide data */
	final boolean amino;
	boolean maskMiddle=false;
	int qHammingDist=0;
	
//	final int maxSupportedK;
	final int bitsPerBase;
	final int maxSymbol;
	final int symbols;
	final int symbolArrayLen;
	final int symbolSpace;
	final long symbolMask;
	
//	final int minlen;
	final int shift;
	final int shift2;
	final long mask;
	final long middleMask;
	
	/** x&clearMasks[i] will clear base i */
	final long[] clearMasks;
	/** x|setMasks[i][j] will set base i to j */
	final long[][] setMasks;
	/** x&leftMasks[i] will clear all bases to the right of i (exclusive) */
	final long[] leftMasks;
	/** x&rightMasks[i] will clear all bases to the left of i (inclusive) */
	final long[] rightMasks;
	
	/** Symbol code; -1 for undefined */
	final byte[] symbolToNumber;
	/** Symbol code; 0 for undefined */
	final byte[] symbolToNumber0;
	/** Complementary symbol code; 0 for undefined */
	final byte[] symbolToComplementNumber0;
	
	/** For verbose / debugging output */
	final String kmerToString(long kmer, int k){
		return amino ? AminoAcid.kmerToStringAA(kmer, k) : AminoAcid.kmerToString(kmer, k);
	}
	
	/** Returns true if the symbol is not degenerate (e.g., 'N') for the alphabet in use. */
	final boolean isFullyDefined(byte symbol){
		return symbol>=0 && symbolToNumber[symbol]>=0;
	}
	
	/*--------------------------------------------------------------*/
	/*----------------        Common Fields         ----------------*/
	/*--------------------------------------------------------------*/
	
	/** Print status messages to this output stream */
	private PrintStream outstream=System.err;
	/** Print verbose messages */
	public static boolean verbose=false;
	/** True if an error was encountered */
	public boolean errorState=false;
	/** Overwrite existing output files */
	private boolean overwrite=false;
	/** Append to existing output files */
	private boolean append=false;
	/** Reads are output in input order */
	private boolean ordered=true;
	
}
