# for emacs: -*- mode: sh; -*-

# This file describes how we made the browser database on
# NCBI build 37 (February 2009 freeze) aka:
#	GRCh37 - Genome Reference Consortium Human Reference 37
#	Assembly Accession: GCA_000001405.1

#	"$Id: hg19.txt,v 1.118 2010/06/10 16:34:40 chinhli Exp $";

#############################################################################

# NOTE FOR NEXT HUMAN ASSEMBLY (2009-07-29 - Brooke): hg19 contains the wrong
# sequence for chrM. The accession NC_001807 was replaced in GenBank with
# NC_012920, with the note: "This sequence was removed since the accepted
# reference sequence for the Homo sapiens mitochondrion is the rCRS/Mitomap
# sequence, which is now available as the record NC_012920".
# Also, from http://www.mitomap.org/mitoseq.html:
# "IMPORTANT:  Do not use NC_001807 as "the rCRS" as it is an African
# (Yoruban) sequence with over 40 variant nucleotides from the rCRS. As of
# July 8, 2009 it has been removed from GenBank as a reference sequence but
# may be found, if needed, as  AF347015, one of 53 African sequence deposited
# in Genbank by Ingman et al in 2001."
# Use NC_012920 for the chrM sequence for the next build!

# Download sequence (DONE - 2009-02-04 - Hiram)
    mkdir -p /hive/data/genomes/hg19/download
    cd /hive/data/genomes/hg19/download
    mkdir -p assembled_chromosomes
    wget --cut-dirs=8 --no-parent --timestamping --no-remove-listing -m \
        --directory-prefix=assembled_chromosomes \
        -nH --ftp-user=anonymous --ftp-password=yourEmail@your.domain \
ftp://ftp.ncbi.nlm.nih.gov/genbank/genomes/Eukaryotes/vertebrates_mammals/Homo_sapiens/GRCh37/Primary_Assembly/assembled_chromosomes

    mkdir -p alternate_loci
for N in 1 2 3 4 5 6 7 8 9
do
wget --cut-dirs=6 --no-parent --timestamping --no-remove-listing -m \
    --directory-prefix=alternate_loci \
        -nH --ftp-user=anonymous --ftp-password=yourEmail@your.domain \
ftp://ftp.ncbi.nlm.nih.gov/genbank/genomes/Eukaryotes/vertebrates_mammals/Homo_sapiens/GRCh37/ALT_REF_LOCI_${N}
done

    mkdir -p unlocalized_scaffolds
    wget --cut-dirs=8 --no-parent --timestamping --no-remove-listing -m \
        --directory-prefix=unlocalized_scaffolds \
	    -nH --ftp-user=anonymous --ftp-password=yourEmail@your.domain \
ftp://ftp.ncbi.nlm.nih.gov/genbank/genomes/Eukaryotes/vertebrates_mammals/Homo_sapiens/GRCh37/Primary_Assembly/unlocalized_scaffolds

    mkdir -p unplaced_scaffolds
    wget --cut-dirs=8 --no-parent --timestamping --no-remove-listing -m \
        --directory-prefix=unplaced_scaffolds \
	    -nH --ftp-user=anonymous --ftp-password=yourEmail@your.domain \
ftp://ftp.ncbi.nlm.nih.gov/genbank/genomes/Eukaryotes/vertebrates_mammals/Homo_sapiens/GRCh37/Primary_Assembly/unplaced_scaffolds

    mkdir -p placed_scaffolds
    wget --cut-dirs=8 --no-parent --timestamping --no-remove-listing -m \
        --directory-prefix=placed_scaffolds \
	    -nH --ftp-user=anonymous --ftp-password=hiram@soe.ucsc.edu \
ftp://ftp.ncbi.nlm.nih.gov/genbank/genomes/Eukaryotes/vertebrates_mammals/Homo_sapiens/GRCh37/Primary_Assembly/placed_scaffolds

    mkdir ucscChr
    cd ucscChr
    for F in ../assembled_chromosomes/FASTA/chr*.fa
do
    C=`basename $F`
    C=${C/.fa}
    echo -n "${C} "
    H=`head -1 "${F}"`
    chrN=`echo $H | sed -e "s/.*Homo sapiens chromosome /chr/; s/, .*//"`
    A=`echo $H | sed -e "s/. Homo.*//; s/.*gb.//"`
    echo $chrN $A
    grep -v "^#" ../assembled_chromosomes/AGP/${chrN}.comp.agp \
        | sed -e "s/^${A}/${chrN}/" > ${chrN}.agp
    echo ">${chrN}" > ${chrN}.fa
    grep -v "^>" ../assembled_chromosomes/FASTA/${chrN}.fa >> ${chrN}.fa
done

    rm -f scaffolds.agp
    find ../alternate_loci -type f | grep ".agp$" | while read F
do
    grep "^GL" $F | sed -e \
"s/^GL000250.1/chr6_apd_hap1/" -e \
"s/^GL000251.1/chr6_cox_hap2/" -e \
"s/^GL000252.1/chr6_dbb_hap3/" -e \
"s/^GL000253.1/chr6_mann_hap4/" -e \ 
"s/^GL000254.1/chr6_mcf_hap5/" -e \
"s/^GL000255.1/chr6_qbl_hap6/" -e \
"s/^GL000256.1/chr6_ssto_hap7/" -e \
"s/^GL000257.1/chr4_ctg9_hap1/" -e \
"s/^GL000258.1/chr17_ctg5_hap1/"
done > scaffolds.agp

    find ../unlocalized_scaffolds -type f | grep ".agp$" \
| while read F
do
    C=`basename ${F}`   
    C=${C/.unlocalized.scaf.agp}
    grep "^GL" ${F} | sed -e "s/^GL\([0-9]*\).1/${C}_gl\1_random/"
done >> scaffolds.agp

    find ../unplaced_scaffolds -type f | grep ".agp$" \
| while read F
do
    grep "^GL" ${F} | sed -e "s/^GL\([0-9]*\).1/chrUn_gl\1/"
done >> scaffolds.agp

    rm -f scaffolds.fa
    find ../alternate_loci -type f | grep ".fa$" | while read F
do  
    sed -e \
"s/>.*GL000250.*/>chr6_apd_hap1/" -e \
"s/>.*GL000251.*/>chr6_cox_hap2/" -e \
"s/>.*GL000252.*/>chr6_dbb_hap3/" -e \
"s/>.*GL000253.*/>chr6_mann_hap4/" -e \
"s/>.*GL000254.*/>chr6_mcf_hap5/" -e \
"s/>.*GL000255.*/>chr6_qbl_hap6/" -e \
"s/>.*GL000256.*/>chr6_ssto_hap6/" -e \
"s/>.*GL000257.*/>chr4_ctg9_hap1/" -e \
"s/>.*GL000258.*/>chr17_ctg5_hap1/" ${F}
done > scaffolds.fa

    find ../unlocalized_scaffolds -type f | grep ".fa$" | while read F
do
    sed -e \
"s/^>.*GL\([0-9]*\).* chromosome \([0-9]*\).*/>chr\2_gl\1_random/" ${F}
done >> scaffolds.fa

    find ../unplaced_scaffolds -type f | grep ".fa$" | while read F
do
    sed -e "s/.*\(GL[0-9]*\).*/\1/; s/GL/>chrUn_gl/" $F
done >> scaffolds.fa


############################################################################
## Create database (DONE - 2009-03-04 - Hiram)
    cd /hive/data/genomes/hg19
    cat << '_EOF_' > hg19.config.ra
# Config parameters for makeGenomeDb.pl:
db hg19
scientificName Homo sapiens
commonName Human
assemblyDate Feb. 2009
assemblyLabel GRCh37 Genome Reference Consortium Human Reference 37 (GCA_000001405.1)
orderKey 14
mitoAcc NC_001807
fastaFiles /hive/data/genomes/hg19/download/ucscChr/*.fa
agpFiles /hive/data/genomes/hg19/download/ucscChr/*.agp
# qualFiles /dev/null
dbDbSpeciesDir human
taxId	9606
'_EOF_'
    # << happy emacs

    time makeGenomeDb.pl hg19.config.ra > makeGenomeDb.log 2>&1
    #	real    14m8.958s
     featureBits -countGaps hg19 gap
    #	239845127 bases of 3137161264 (7.645%) in intersection
    featureBits -noRandom -noHap -countGaps hg19 gap
    #	234344806 bases of 3095693983 (7.570%) in intersection
    #	verify featureBits is properly ignorning haps and randoms:
    egrep -v "_" chrom.sizes | awk '{sum+=$2;print sum,$0}'
    #	3095693983 chrM 16571
    #	same total as in featureBits

    #	much later on, discovered that we needed a chrM definition in the
    #	agp files, added by hand to hg19/M/chrM.agp and hg19/hg19.agp the line:
# chrM    1       16571   1       F       NC001807        1       16571   +
    #	the spaces there are tabs

############################################################################
# running repeat masker (DONE - 2009-03-05 - Hiram)
    screen # use screen to manage this day-long job
    mkdir /hive/data/genomes/hg19/bed/repeatMasker
    cd /hive/data/genomes/hg19/bed/repeatMasker
    time doRepeatMasker.pl -bigClusterHub=swarm -buildDir=`pwd` hg19 \
	> do.log 2>&1
    #	real    525m23.521s
    cat faSize.rmsk.txt
    #	3137161264 bases (239850802 N's 2897310462 real 1431585691
    #	upper 1465724771 lower) in 93 sequences in 1 files
    #	%46.72 masked total, %50.59 masked real
    featureBits -countGaps hg19 rmsk
    #	1465724774 bases of 3137161264 (46.721%) in intersection
    #	this is odd, 3 bases more in featureBits than were masked ?
    #	check it out, make a bed file from the featureBits:
    featureBits -countGaps -bed=rmsk.bed hg19 rmsk
    #	went down a sequence of intersections with this idea, but could
    #	not get it resolved.  It appears there are 75 bases in the rmsk
    #	table that were not masked in the 2bit file ?
    #	Later on, realized that featureBits does not count lower case N's
    #	in the "lower" category, but only in the N's category.

    #	trying a non-split table:
    hgsql -e "show tables;" hg19 | grep _rmsk | while read T
do
    hgsql -e "drop table ${T};" hg19
done
    hgLoadOut -nosplit -verbose=2 -table=rmsk hg19 hg19.fa.out
bad rep range [4385, 4384] line 1348605 of hg19.fa.out 
bad rep range [5563, 5562] line 1563988 of hg19.fa.out 
bad rep range [4539, 4538] line 3111186 of hg19.fa.out 
    #	featureBits still reports 1465724774 bases in rmsk table
    #	cleaning the hg19.fa.out file:
    cp hg19.fa.out hg19.clean.out
    # edit hg19.clean.out and remove the three lines:
# 1467  20.7  1.2 17.6  chr14     35056767 35056794 (72292746) +  L1ME1          LINE/L1               4385 4384 (1761) 1120962
# 1943  23.8  5.0 12.6  chr15     65775909 65775924 (36755468) +  L1MC4          LINE/L1               5563 5562 (2480) 1299299
# 2463  25.1  5.0 11.6  chr3      121291056 121291083 (76731347) +  L1M3           LINE/L1               4539 4538 (1608) 2589267

    #	reload the table
    hgsql -e "drop table rmsk;" hg19
    hgLoadOut -nosplit -verbose=2 -table=rmsk hg19 hg19.clean.out

    #	try masking with this clean file:
    twoBitMask /hive/data/genomes/hg19/hg19.unmasked.2bit hg19.clean.out \
	hg19.clean.2bit
    twoBitToFa hg19.clean.2bit stdout | faSize stdin > faSize.clean.txt
    cat faSize.clean.txt
    #	this gives the lower by 75 bases result:
    #	3137161264 bases (239850802 N's 2897310462 real 1431585763 upper
    #	1465724699 lower) in 93 sequences in 1 files
    #	%46.72 masked total, %50.59 masked real
    featureBits -countGaps hg19 rmsk
    #	1465724774 bases of 3137161264 (46.721%) in intersection
    #	is the countGaps interferring ?
    featureBits hg19 rmsk
    #	1465724774 bases of 2897316137 (50.589%) in intersection
    #	nope, lets' see what the .out file has:
    grep chr hg19.clean.out | sed -e "s/^  *//" | awk '{print $5,$6-1,$7}' \
	| sort -k1,1 -k2,2n > hg19.clean.out.bed
    featureBits -countGaps hg19 hg19.clean.out.bed
    #	1465724774 bases of 3137161264 (46.721%) in intersection
    #	is it perhaps not masking N's ?
    twoBitToFa hg19.clean.2bit stdout | grep n | less
    #	that does find some lower case n's, find all N's:
    findMotif -strand=+ -motif=gattaca -verbose=4 hg19.clean.2bit \
	2> findMotif.out
    grep "^#GAP" findMotif.out | sed -e "s/#GAP //" > nLocations.bed
    #	which cover:
    featureBits -countGaps hg19 nLocations.bed
    #	251299071 bases of 3137161264 (8.010%) in intersection
    #	overlapping rmsk business with these N locations:
    featureBits -countGaps hg19 hg19.clean.out.bed nLocations.bed
    #	6494740 bases of 3137161264 (0.207%) in intersection
    #	and overlapping with gap:
    featureBits -countGaps hg19 gap nLocations.bed
    #	239845127 bases of 3137161264 (7.645%) in intersection

############################################################################
# running TRF simple repeats (DONE - 2009-03-05 - Hiram)
    screen # use screen to manage this day-long job
    mkdir /hive/data/genomes/hg19/bed/simpleRepeat
    cd /hive/data/genomes/hg19/bed/simpleRepeat
    time doSimpleRepeat.pl -bigClusterHub=pk -workhorse=hgwdev \
	-smallClusterHub=pk -buildDir=`pwd` hg19 > do.log 2>&1
    #	real    33m25.815s

    twoBitMask bed/repeatMasker/hg19.clean.2bit \
	-add bed/simpleRepeat/trfMask.bed hg19.2bit
    twoBitToFa hg19.2bit stdout | faSize stdin > faSize.hg19.2bit.txt
# 3137161264 bases (239850802 N's 2897310462 real 1430387259 upper
# 1466923203 lower) in 93 sequences in 1 files
# %46.76 masked total, %50.63 masked real

############################################################################
#	prepare cluster data (DONE - 2009-03-06 - Hiram)
    cd /hive/data/genomes/hg19
    rm /gbdb/hg19/hg19.2bit
    ln -s `pwd`/hg19.2bit /gbdb/hg19/hg19.2bit

    time blat hg19.2bit \
	/dev/null /dev/null -tileSize=11 -makeOoc=11.ooc -repMatch=1024
    #	Wrote 30675 overused 11-mers to 11.ooc
    #	real    3m11.302s

    mkdir /hive/data/staging/data/hg19
    cp -p hg19.2bit /hive/data/staging/data/hg19
    cp -p 11.ooc /hive/data/staging/data/hg19
    cp -p chrom.sizes /hive/data/staging/data/hg19

    mkdir separateChrs
    cd separateChrs
    grep -v "_" ../chrom.sizes | awk '{print $1}' | while read C
do
    twoBitToFa -seq="${C}" ../hg19.2bit stdout
done | faToTwoBit stdin hg19.chrOnly.2bit
    twoBitInfo hg19.chrOnly.2bit stdout | sort -k2,2nr > chrOnly.chrom.sizes

    grep "_hap" ../chrom.sizes | awk '{print $1}' | while read C
do
    twoBitToFa -seq="${C}" ../hg19.2bit stdout
done | faToTwoBit stdin hg19.hapOnly.2bit
    twoBitInfo hg19.hapOnly.2bit stdout | sort -k2,2nr > hapOnly.chrom.sizes

    grep "_" ../chrom.sizes | grep -v "_hap" | awk '{print $1}' | while read C
do
    twoBitToFa -seq="${C}" ../hg19.2bit stdout
done | faToTwoBit stdin hg19.scaffolds.2bit
    twoBitInfo hg19.scaffolds.2bit stdout | sort -k2,2nr > scaffolds.chrom.sizes

    cp -p *.2bit *.sizes /hive/data/staging/data/hg19

    # ask admin to sync this directory: /hive/data/staging/data/hg19/
    #	to the kluster nodes /scratch/data/hg19/

############################################################################
# running cpgIsland business (DONE - 2009-03-06 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/cpgIsland
    cd /hive/data/genomes/hg19/bed/cpgIsland
    cvs -d /projects/compbio/cvsroot checkout -P hg3rdParty/cpgIslands
    cd hg3rdParty/cpgIslands
    # comment out the following two lines if it compiles cleanly
    # some day  (there were some other fixups too, adding include lines)
    sed -e "s#\(extern char\* malloc\)#// \1#" cpg_lh.c > tmp.c
    mv tmp.c cpg_lh.c
    make
    cd ../../ 
    ln -s hg3rdParty/cpgIslands/cpglh.exe
    mkdir -p hardMaskedFa
    cut -f1 ../../chrom.sizes | while read C
do
    echo ${C}
    twoBitToFa ../../hg19.2bit:$C stdout \
	| maskOutFa stdin hard hardMaskedFa/${C}.fa
done

    cut -f1 ../../chrom.sizes > chr.list
    cat << '_EOF_' > template
#LOOP
./runOne $(root1) {check out line results/$(root1).cpg}
#ENDLOOP
'_EOF_'
    # << happy emacs

    cat << '_EOF_' > runOne
#!/bin/csh -fe
./cpglh.exe hardMaskedFa/$1.fa > /scratch/tmp/$1.$$
mv /scratch/tmp/$1.$$ $2
'_EOF_'
    # << happy emacs

    gensub2 chr.list single template jobList
    para create jobList
    para try
    para check ... etc
    para time
# Completed: 93 of 93 jobs
# CPU time in finished jobs:        172s       2.86m     0.05h    0.00d  0.000 y
# IO & Wait Time:                  1748s      29.14m     0.49h    0.02d  0.000 y
# Average job time:                  21s       0.34m     0.01h    0.00d
# Longest finished job:              34s       0.57m     0.01h    0.00d
# Submission to last job:            83s       1.38m     0.02h    0.00d

    # Transform cpglh output to bed +
    catDir results | awk '{
$2 = $2 - 1;
width = $3 - $2;
printf("%s\t%d\t%s\t%s %s\t%s\t%s\t%0.0f\t%0.1f\t%s\t%s\n",
       $1, $2, $3, $5,$6, width,
       $6, width*$7*0.01, 100.0*2*$6/width, $7, $9);
}' > cpgIsland.bed

    cd /hive/data/genomes/hg19/bed/cpgIsland
    hgLoadBed hg19 cpgIslandExt -tab \
      -sqlTable=$HOME/kent/src/hg/lib/cpgIslandExt.sql cpgIsland.bed

# Reading cpgIsland.bed
# Loaded 28226 elements of size 10
# Sorted
# Saving bed.tab
# Loading hg19

############################################################################
# create lift file on unBridged gaps for genbank splits (2009-03-09 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/gap
    cd /hive/data/genomes/hg19/bed/gap
    gapToLift hg19 hg19.unBridged.lift -bedFile=unBridged.lift.bed
    cp -p hg19.unBridged.lift ../../jkStuff
    cp -p hg19.unBridged.lift /hive/data/staging/data/hg19

############################################################################
# AUTO UPDATE GENBANK RUN  (DONE - 2009-03-07,13 - Hiram)
    # align with latest genbank process.
    cd ~/kent/src/hg/makeDb/genbank
    cvsup
    # edit etc/genbank.conf to add hg19 just after hg18

# hg19 - GRCh37 - Genome Reference Consortium Human Reference 37
#       Assembly Accession: GCA_000001405.1
hg19.serverGenome = /hive/data/genomes/hg19/hg19.2bit
hg19.clusterGenome = /scratch/data/hg19/hg19.2bit
hg19.ooc = /scratch/data/hg19/11.ooc
hg19.lift = /hive/data/genomes/hg19/jkStuff/hg19.unBridged.lift
hg19.hapRegions = /hive/data/genomes/hg19/jkStuff/hg19.haplotypes.psl
hg19.refseq.mrna.native.pslCDnaFilter  = ${finished.refseq.mrna.native.pslCDnaFilter}
hg19.refseq.mrna.xeno.pslCDnaFilter    = ${finished.refseq.mrna.xeno.pslCDnaFilter}
hg19.genbank.mrna.native.pslCDnaFilter = ${finished.genbank.mrna.native.pslCDnaFilter}
hg19.genbank.mrna.xeno.pslCDnaFilter   = ${finished.genbank.mrna.xeno.pslCDnaFilter}
hg19.genbank.est.native.pslCDnaFilter = ${finished.genbank.est.native.pslCDnaFilter}
hg19.genbank.est.xeno.pslCDnaFilter   = ${finished.genbank.est.xeno.pslCDnaFilter}
hg19.genbank.est.xeno.load = yes
hg19.refseq.mrna.xeno.load  = yes
hg19.refseq.mrna.xeno.loadDesc = yes
hg19.mgc = yes
hg19.orfeome = yes
hg19.downloadDir = hg19
hg19.ccds.ncbiBuild = 37.1
hg19.upstreamGeneTbl = refGene
hg19.upstreamMaf = multiz46way /hive/data/genomes/hg19/bed/multiz46way/species.list
hg19.genbank.mrna.blatTargetDb = yes
hg19.perChromTables = no

    cvs ci -m "Added hg19." etc/genbank.conf
    # update /cluster/data/genbank/:
    make etc-update

    ssh genbank
    screen		#	use a screen to manage this job
    cd /cluster/data/genbank
    time nice -n +19 bin/gbAlignStep -initial hg19 &
    #	logFile: var/build/logs/2009.03.10-20:28:44.hg19.initalign.log
    #	real    2761m13.680s
    #	that ran on the swarm with little interference and no problems

    # load database when finished
    ssh hgwdev
    screen	# use screen to manage this long running command
    cd /cluster/data/genbank
    time nice -n +19 ./bin/gbDbLoadStep -drop -initialLoad hg19 &
    # logFile: var/dbload/hgwdev/logs/2009.03.12-21:10:02.dbload.log
    #	real    369m11.941s

    # enable daily alignment and update of hgwdev (DONE - 2009-02-24 - Hiram)
    cd ~/kent/src/hg/makeDb/genbank
    cvsup
    # add hg19 to:
        etc/align.dbs
        etc/hgwdev.dbs
    cvs ci -m "Added hg19 - Human - GRCh37" etc/align.dbs etc/hgwdev.dbs
    make etc-update

#########################################################################
#  BLATSERVERS ENTRY (DONE - 2009-03-09 - Hiram)
#	After getting a blat server assigned by the Blat Server Gods,
    ssh hgwdev

    hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
	VALUES ("hg19", "blat13", "17778", "1", "0"); \
	INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
	VALUES ("hg19", "blat13", "17779", "0", "1");' \
	    hgcentraltest
    #	test it with some sequence

############################################################################
# Making download files (DONE - 2009-03-13 - Hiram)
    cd /hive/data/genomes/hg19
    makeDownloads.pl -allowMissedTrfs -noChromRoot hg19 \
	> downloads.log 2>&1
############################################################################
# Venter1 chain, net experiment (DONE - Hiram - 2009-03-15)
doBlastzChainNet.pl `pwd`/DEF \
        -stop=partition -bigClusterHub=swarm \
        -smallClusterHub=swarm -chainMinScore=1000 -chainLinearGap=medium \
        -workhorse=hgwdev -fileServer=hgwdev > partition.log 2>&1

doBlastzChainNet.pl `pwd`/DEF \
        -continue=blastz -stop=blastz -bigClusterHub=swarm \
        -smallClusterHub=swarm -chainMinScore=1000 -chainLinearGap=medium \
        -workhorse=hgwdev -fileServer=hgwdev > blastz.log 2>&1

doBlastzChainNet.pl `pwd`/DEF \
        -continue=cat -stop=net -bigClusterHub=swarm \
        -smallClusterHub=swarm -chainMinScore=1000 -chainLinearGap=medium \
        -workhorse=hgwdev -fileServer=hgwdev > net.log 2>&1
real    163m28.438s

    # to load, run it in debug, then check the load script
doBlastzChainNet.pl `pwd`/DEF \
	-noLoadChainSplit -continue=load -stop=load -bigClusterHub=swarm \
	-debug -smallClusterHub=swarm -chainMinScore=1000 \
	-chainLinearGap=medium \
	-workhorse=hgwdev -fileServer=hgwdev > load.log 2>&1

    # and create a synNet for multiz, run in debug, and examine script
    #	to make sure it works correctly
doBlastzChainNet.pl `pwd`/DEF \
	-syntenicNet -continue=syntenicNet -stop=syntenicNet \
	-debug -bigClusterHub=swarm \
	-smallClusterHub=swarm -chainMinScore=1000 -chainLinearGap=medium \
	-workhorse=hgwdev -fileServer=hgwdev > synNet.log 2>&1
    #	real    31m11.216s

############################################################################
# reset position to chr6 haplotype situation
    hgsql -e \
'update dbDb set defaultPos="chr6:28343766-33555363" where name="hg19";' \
	hgcentraltest

# reset to a smaller range (2009-04-24 - Brooke)
# this is the SOD1 gene, implicated in Lou Gehrig's disease.

    hgsql -e \
'update dbDb set defaultPos="chr21:33,031,597-33,041,570" where name="hg19";' \
        hgcentraltest

############################################################################
# Self Lastz run (DONE - 2009-03-19 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzSelf.2009-03-19
    cd /hive/data/genomes/hg19/bed/lastzSelf.2009-03-19
    cat << '_EOF_'
# human vs human
BLASTZ=lastz
# maximum M allowed with lastz is only 255
BLASTZ_M=254
# lastz does not like the O= and E= lines in the matrix file 
#       this copy has that removed from /scratch/data/scratch/human_chimp.v2.q
BLASTZ_Q=/hive/data/genomes/hg19/bed/lastzHg19Haps.2009-03-09/human_chimp.v2.q
# and place those items here
BLASTZ_O=600
BLASTZ_E=150
# other parameters from hg18 vs venter1 lastz on advice from Webb
BLASTZ_K=10000
BLASTZ_Y=15000
BLASTZ_T=2

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_IN_CONTIGS=0
    
# QUERY: Human Hg19
SEQ2_DIR=/scratch/data/hg19/hg19.2bit
SEQ2_LEN=/scratch/data/hg19/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0
SEQ2_IN_CONTIGS=0
    
BASE=/hive/data/genomes/hg19/bed/lastzSelf.2009-03-19
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    screen # use screen to manage this long-running job
    time nice -n +19 doBlastzChainNet.pl `pwd`/DEF -verbose=2 \
	-noLoadChainSplit -chainMinScore=2000 -chainLinearGap=medium \
	-workhorse=hgwdev \
	-stop=net -smallClusterHub=pk -bigClusterHub=swarm > do.log 2>&1 &
    #	cluster difficulties, finished manually, then:
    time nice -n +19 doBlastzChainNet.pl `pwd`/DEF -verbose=2 \
	-noLoadChainSplit -chainMinScore=2000 -chainLinearGap=medium \
	-continue=cat -workhorse=hgwdev \
	-stop=net -smallClusterHub=pk -bigClusterHub=swarm > cat.log 2>&1 &

    time nice -n +19 doBlastzChainNet.pl `pwd`/DEF -verbose=2 \
	-noLoadChainSplit -chainMinScore=2000 -chainLinearGap=medium \
	-continue=load -debug -workhorse=hgwdev \
	-stop=load -smallClusterHub=pk -bigClusterHub=swarm > load.debug.log 2>&1 &
    #	that indicates it would do:
    hgLoadChain -tIndex hg19 chainSelf hg19.hg19.all.chain.gz
    #	adding -normScore
    hgLoadChain -normScore -tIndex hg19 chainSelf hg19.hg19.all.chain.gz

    # a user asked about axtNet files, d
    time nice -n +19 doBlastzChainNet.pl `pwd`/DEF -verbose=2 \
	-noLoadChainSplit -chainMinScore=2000 -chainLinearGap=medium \
	-ignoreSelf -continue=net -workhorse=hgwdev \
	-stop=net -smallClusterHub=encodek -bigClusterHub=swarm > net.log 2>&1 &
    #	about 8m 17s
    cd /hive/data/genomes/hg19/bed/lastzSelf.2009-03-19/axtChain
    netClass -verbose=0 -noAr noClass.net hg19 hg19 hg19.hg19.net
    gzip hg19.hg19.net
    cd /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/vsSelf
    ln -s \
/hive/data/genomes/hg19/bed/lastzSelf.2009-03-19/axtChain/hg19.hg19.net.gz .
    # fixup README.txt and md5sum.txt files
    md5sum hg19.hg19.net.gz >> md5sum.txt
    # Brian wants to see the track:
    netFilter -minGap=10 hg19.hg19.net.gz \
	| hgLoadNet -verbose=0 hg19 netSelf stdin

############################################################################
# Chimp Lastz run (DONE - 2009-03-19 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzPanTro2.2009-03-19
    cd /hive/data/genomes/hg19/bed/lastzPanTro2.2009-03-19
    cat << '_EOF_'
# human vs chimp
BLASTZ=lastz
# maximum M allowed with lastz is only 254
BLASTZ_M=254
# lastz does not like the O= and E= lines in the matrix file
#       this copy has that removed from /scratch/data/scratch/human_chimp.v2.q
BLASTZ_Q=/hive/data/genomes/hg19/bed/lastzHg19Haps.2009-03-09/human_chimp.v2.q
# and place those items here
BLASTZ_O=600
BLASTZ_E=150
# other parameters from panTro2 vs hg18 lastz on advice from Webb
BLASTZ_K=4500
BLASTZ_Y=15000
BLASTZ_T=2

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_IN_CONTIGS=0

# QUERY: Chimp PanTro2
SEQ2_DIR=/scratch/data/panTro2/panTro2.2bit
SEQ2_LEN=/scratch/data/panTro2/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0
SEQ2_IN_CONTIGS=0

BASE=/hive/data/genomes/hg19/bed/lastzPanTro2.2009-03-19
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    screen # use screen to manage this long-running job
    time nice -n +19 doBlastzChainNet.pl `pwd`/DEF -verbose=2 \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=pk -bigClusterHub=swarm > do.log 2>&1 &
    #	real    173m22.880s
    #	cluster problems, continuing after lastz done:
    time nice -n +19 doBlastzChainNet.pl `pwd`/DEF -verbose=2 -continue=cat \
	-stop=net -noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=pk -bigClusterHub=swarm \
	> net.log 2>&1 &
    #	real    81m20.209s
    #	continuing with the load and adding syntenicNet
    time nice -n +19 doBlastzChainNet.pl `pwd`/DEF -verbose=2 -continue=load \
	-syntenicNet -noLoadChainSplit -chainMinScore=5000 \
	-chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=pk -bigClusterHub=swarm \
	> load.log 2>&1 &
    #	real    47m17.871s
    cat fb.hg19.chainPanTro2Link.txt 
    #	2747983350 bases of 2897316137 (94.846%) in intersection

    #	running the swap - DONE - 2009-05-24
    ssh swarm
    mkdir /hive/data/genomes/panTro2/bed/blastz.hg19.swap
    cd /hive/data/genomes/panTro2/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	-swap /hive/data/genomes/hg19/bed/lastzPanTro2.2009-03-19/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=swarm -bigClusterHub=swarm \
	> swap.log 2>&1 &
    #	real    723m41.377s
    cat fb.panTro2.chainHg19Link.txt 
    #	2761343871 bases of 2909485072 (94.908%) in intersection

############################################################################
# Creating the pushQ entry (DONE - 2009-03-20 - Hiram)
    mkdir /hive/data/genomes/hg19/pushQ
    cd /hive/data/genomes/hg19/pushQ
    makePushQSql.pl hg19 > hg19.pushQ.sql 2> make.err
    # many complaints about the chain and net tables from the haplotype
    #	experiments, and this table:
    #	orfeomeGenes
    #	which is probably in genbank, and these usual ones:
    #	hg19 does not have seq
    #	hg19 does not have extFile

############################################################################
# Determine PAR region of X and Y (DONE - 2009-03-20 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/parRegion
    cd /hive/data/genomes/hg19/bed/parRegion
    awk '$5 != "N"' ../../X/chrX.agp | awk '{print $6}' | sort > chrX.cloneList
    awk '$5 != "N"' ../../Y/chrY.agp | awk '{print $6}' | sort > chrY.cloneList
    comm -12 chrX.cloneList chrY.cloneList > chrXY.par.clone.list
    cat chrXY.par.clone.list \
	| while read C; do grep "${C}" ../../X/chrX.agp; done \
	| sort -k1,1 -k2,2n >> chrX.par.region.agp
    cat chrXY.par.clone.list \
	| while read C; do grep "${C}" ../../Y/chrY.agp; done \
	| sort -k1,1 -k2,2n >> chrY.par.region.agp
    awk '{printf "%s\t%d\t%d\t%s\n", $1, $2-1, $3, $6}' chrY.par.region.agp \
	> chrY.par.region.bed
    awk '{printf "%s\t%d\t%d\t%s\n", $1, $2-1, $3, $6}' chrX.par.region.agp \
	> chrX.par.region.bed
    #	use those bed files in custom tracks on hg19 to verify that they
    #	are two continuous regions with only gaps between these items
    #	these location extents are: (zero relative)
    #	chrX 60000 2722842
    #	chrX 154906585 155260560
    #	chrY 10000 2649520
    #	chrY 59034049 59363566

############################################################################
# Gorilla Lastz run (DONE - 2009-03-21,05-13 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzGorGor1.2009-03-21
    cd /hive/data/genomes/hg19/bed/lastzGorGor1.2009-03-21
    cat << '_EOF_'
# human vs gorilla
BLASTZ=lastz
# maximum M allowed with lastz is only 254
BLASTZ_M=254
# lastz does not like the O= and E= lines in the matrix file
#       this copy has that removed from /scratch/data/scratch/human_chimp.v2.q
BLASTZ_Q=/hive/data/genomes/hg19/bed/lastzHg19Haps.2009-03-09/human_chimp.v2.q
# and place those items here
BLASTZ_O=600
BLASTZ_E=150
# other parameters from panTro2 vs hg18 lastz on advice from Webb
BLASTZ_K=4500
BLASTZ_Y=15000
BLASTZ_T=2

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=100000000
SEQ1_LAP=10000
SEQ1_IN_CONTIGS=0

# QUERY: Gorilla gorGor1
SEQ2_DIR=/scratch/data/gorGor1/gorGor1.2bit
SEQ2_LEN=/scratch/data/gorGor1/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LIMIT=300
SEQ2_LAP=0
SEQ2_IN_CONTIGS=0

BASE=/hive/data/genomes/hg19/bed/lastzGorGor1.2009-03-21
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    screen # use screen to manage this long-running job
    time nice -n +19 doBlastzChainNet.pl `pwd`/DEF -verbose=2 \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=pk -bigClusterHub=swarm \
	> do.log 2>&1 &
    cat fb.hg19.chainGorGor1Link.txt 
    #	1723432141 bases of 2897316137 (59.484%) in intersection
    doRecipBest.pl -buildDir=`pwd` hg19 gorGor1 > rbest.log 2>&1

############################################################################
# PREPARE LINEAGE SPECIFIC REPEAT FILES FOR LASTZ (DONE - 2009-04-02 - Hiram)
    ssh pk
    mkdir /hive/data/genomes/hg19/bed/linSpecRep
    cd /hive/data/genomes/hg19/bed/linSpecRep
    #	create individual .out files from the master record in ../repeatMasker
    mkdir splitOut
    cat << '_EOF_' > split.csh
#!/bin/csh -fe
set C = $1
head -3 ../repeatMasker/hg19.clean.out > splitOut/${C}.out
grep "${C} " ../repeatMasker/hg19.clean.out >> splitOut/${C}.out
'_EOF_'
    # << happy emacs

    cat << '_EOF_' > template
#LOOP
split.csh $(root1) {check out line+ splitOut/$(root1).out}
#ENDLOOP
'_EOF_'
    # << happy emacs

    cut -f1 ../../chrom.sizes > chrom.list
    gensub2 chrom.list single template jobList
    para create jobList
    para try ... check ... push ... etc...
# Completed: 93 of 93 jobs
# CPU time in finished jobs:        127s       2.12m     0.04h    0.00d  0.000 y
# IO & Wait Time:                 17154s     285.90m     4.76h    0.20d  0.001 y
# Average job time:                 186s       3.10m     0.05h    0.00d
# Longest finished job:             224s       3.73m     0.06h    0.00d
# Submission to last job:           280s       4.67m     0.08h    0.00d
    
    #	now, we can date and process each of those .out files
    #	this really should be a single creation of notInOthers
    #	These four different ones all end up to be the same anyhow
    #	the notInMouse becomes notInOthers below and the others are removed.
    mkdir dateRepeats
    cd dateRepeats
    cat << '_EOF_' > mkLSR
#!/bin/csh -fe
rm -f $1.out_mus-musculus_rattus_canis-familiaris_bos-taurus
ln -s ../splitOut/$1.out .
/scratch/data/RepeatMasker/DateRepeats \
    $1.out -query human -comp mouse -comp rat -comp dog -comp cow
rm $1.out
mkdir -p ../notInMouse ../notInRat ../notInDog ../notInCow
/cluster/bin/scripts/extractRepeats 1 $1.out_mus*-taurus \
	> ../notInMouse/$1.out.spec
/cluster/bin/scripts/extractRepeats 2 $1.out_mus*-taurus \
	> ../notInRat/$1.out.spec
/cluster/bin/scripts/extractRepeats 3 $1.out_mus*-taurus \
	> ../notInDog/$1.out.spec
/cluster/bin/scripts/extractRepeats 4 $1.out_mus*-taurus \
	> ../notInCow/$1.out.spec
'_EOF_'
    #	<< happy emacs
    chmod +x mkLSR

    cat << '_EOF_' > template
#LOOP
./mkLSR $(path1) {check out line+ $(path1).out_mus-musculus_rattus_canis-familiaris_bos-taurus}
#ENDLOOP
'_EOF_'
    #	<< happy emacs

    gensub2 ../chrom.list single template jobList
    para try ... check ... push ... etc...
    para time
# Completed: 93 of 93 jobs
# CPU time in finished jobs:       2441s      40.69m     0.68h    0.03d  0.000 y
# IO & Wait Time:                   332s       5.53m     0.09h    0.00d  0.000 y
# Average job time:                  30s       0.50m     0.01h    0.00d
# Longest finished job:             125s       2.08m     0.03h    0.00d
# Submission to last job:           454s       7.57m     0.13h    0.01d

    done

    #	these four types of out.spec results all turn out to be identical
    #	To check identical
    cd /hive/data/genomes/hg19/bed/linSpecRep
    find . -name "*.out.spec" | \
	while read FN; do echo `cat ${FN} | sum -r` ${FN}; done \
	| sort -k1,1n | sort -t"/" -k3,3 | sed -e "s#./notIn.*/##" \
	| sort | uniq -c | less
    #	You will see they are all a count of 4
    #	Set them up on scratch data and get to all the kluster nodes:
    mkdir /hive/data/staging/data/hg19/lineageSpecificRepeats
    cd notInMouse
    rsync -a --progress ./ /hive/data/staging/data/hg19/lineageSpecificRepeats
    cd ..
    mv notInMouse notInOthers
    #	do not need to keep all of these
    rm -fr notInRat notInDog notInCow

    # We also need the nibs for blastz runs with lineage specific repeats
    mkdir /hive/data/genomes/hg19/bed/nibs
    cd /hive/data/genomes/hg19/bed/nibs
    cut -f1 ../../chrom.sizes | while read C
do
    twoBitToFa -seq=${C} ../../hg19.2bit stdout \
	| faToNib -softMask stdin ${C}.nib
    echo "${C} done"
done
    mkdir /hive/data/staging/data/hg19/nib
    rsync -a --progress ./ /hive/data/staging/data/hg19/nib

    # Ask cluster-admin to sync /scratch/ filesystem to kluster nodes

#############################################################################
# create gc5Base download file (DONE - 2009-04-24 - Hiram)
    cd /hive/data/genomes/hg19/bed/gc5Base
    hgGcPercent -wigOut -doGaps -file=stdout -win=5 -verbose=0 hg19 \
        /cluster/data/hg19/hg19.2bit | gzip -c > hg19.gc5Base.txt.gz

#############################################################################
# Physical Map Contigs - ctgPos (DONE - 2009-04-23 - Hiram) (Alt. haplotypes added 4/12/10 angie)
    mkdir /hive/data/genomes/hg19/bed/ctgPos
    cd /hive/data/genomes/hg19/bed/ctgPos
    cat << '_EOF_' > mkCtgPos.sh
AGP="/hive/data/genomes/hg19/download/assembled_chromosomes/AGP"
export AGP
for F in `(cd ${AGP}; ls chr*.agp | grep -v ".comp.agp")`
do
    C=${F/.agp/}
    grep "^CM" "${AGP}/${F}" | awk '$5 != "N"' | awk '
{
printf "%s\t%d\t%s\t%d\t%d\n", $6, $8-$7+1, "'${C}'", $2-1+$7-1, $2-1+$8
}
'
done
'_EOF_'
    # << happy emacs
    chmod +x mkCtgPos.sh
    ./mkCtgPos.sh > ctgPos.tab

    cat << '_EOF_' > mkRanCtgPos.sh
AGP="/hive/data/genomes/hg19/download/unlocalized_scaffolds/AGP"
export AGP
for F in `(cd ${AGP}; ls chr*.agp)`
do
    C=${F/.unlocalized.scaf.agp/}
    c=${C/chr/}
    export C c
    grep "^GL" "${AGP}/${F}" | awk '$5 != "N"' | awk '
BEGIN {
    ctgName=""
    ctgStart=0
    ctgEnd=0
    chrom="'${c}'"
    ctgNameLower=""
}
{
if (match(ctgName,$1)) {
    ctgEnd = $3
} else {
    if (length(ctgName) > 0) {
        size=ctgEnd - ctgStart
printf "%s\t%d\tchr%s_%s_random\t%d\t%d\n", ctgName, size, chrom, ctgNameLower, 
ctgStart, ctgEnd
    }
    ctgStart = $2 - 1
    ctgEnd = $3
    ctgName = $1
    ctgNameLower = tolower($1)
    sub(".1$","",ctgNameLower)
}
}
END {
size=ctgEnd - ctgStart
printf "%s\t%d\tchr%s_%s_random\t%d\t%d\n", ctgName, size, chrom, ctgNameLower, 
ctgStart, ctgEnd
}
'
done
'_EOF_'
    # << happy emacs
    chmod +x mkRanCtgPos.sh
    ./mkRanCtgPos.sh >> ctgPos.tab

    #	fetch .sql definition from hg18
    chmod 777 .
    hgsqldump --all -c --tab=. hg18 ctgPos
    # Don't confuse us w/hg18 data:
    rm ctgPos.txt
    chmod 775 .
    hgsql hg19 < ctgPos.sql
    hgsql -e 'load data local infile "ctgPos.tab" into table ctgPos;' hg19

    # 4/12/10 (angie): add the alt loci:
    perl -we 'while (<>) { \
                next if (/^#/); chomp; @w = split; \
                $w[0] = lc($w[0]); $w[0] =~ s/^hs//; $w[0] =~ s/_mhc_/_/;  $w[0] =~ s/_ctg1$//; \
                $w[0] =~ s/_apd$/_apd_hap1/; $w[0] =~ s/_cox$/_cox_hap2/; \
                $w[0] =~ s/_dbb$/_dbb_hap3/; $w[0] =~ s/_mann$/_mann_hap4/; \
                $w[0] =~ s/_mcf$/_mcf_hap5/; $w[0] =~ s/_qbl$/_qbl_hap6/; \
                $w[0] =~ s/_ssto$/_ssto_hap7/; \
                $w[0] =~ s/_1(_ctg\d)/${1}_hap1/; \
                if ($w[0] eq "chr6_cox_hap2" && $w[8] == 4873745) { $w[8] = 4795371; } # yep, inconsistent \
                print join("\t", $w[1], $w[8], $w[0], 0, $w[8]) . "\n"; }' \
    /hive/data/genomes/hg19/download/alternate_loci/*/placed_scaffolds/alt_locus_scaf2primary.pos \
      >> ctgPos.tab
    sort -k 3,3 -k4n,4n ctgPos.tab \
    | hgLoadSqlTab hg19 ctgPos ctgPos.sql stdin
    # TODO: tell NCBI alternate_loci/ALT_REF_LOCI_2/placed_scaffolds/alt_locus_scaf2primary.pos
    # has size inconsistent w/AGP, FASTA


#############################################################################
# CLONE ENDS - first step for BACEND/CytoBand tracks
#	(DONE - 2009-04-28 - Hiram)
    mkdir -p /hive/data/genomes/hg19/bed/cloneend/ncbi
    cd /hive/data/genomes/hg19/bed/cloneend/ncbi

    wget --timestamping \
'ftp://ftp.ncbi.nih.gov/genomes/CLONEEND/homo_sapiens/9606_clone_ends*.mfa.gz'
    wget --timestamping \
'ftp://ftp.ncbi.nih.gov/genomes/CLONEEND/homo_sapiens/9606_clone_info*.txt.gz'

    cd /hive/data/genomes/hg19/bed/cloneend
    # seems like the *.mfa files were split just for convenience
    # concatenate

    for F in ncbi/*.mfa.gz
do
    zcat "${F}"
    echo "${F}" 1>&2
done | gzip > all.mfa.gz
    #	that 1>&2 echos to stderr so you can see the file name and not
    #	interfere with the pipe stdout output to gzip

    # Convert the title line of the all.mfa file
    zcat all.mfa.gz \
	| sed -e "s#^>gi.[0-9]*.gb.#>#; s#^>gi.[0-9]*.emb.#>#; s#\.[0-9]|.*##" \
	    | gzip > cloneEnds.fa.gz

    zcat all.mfa | ./convert.pl | gzip > cloneEnds.fa.gz

    #	make sure nothing got broken:
    faSize all.mfa.gz
# 400901385 bases (5941742 N's 394959643 real 255835696 upper 139123947 lower)
# in 833173 sequences in 1 files

    faSize cloneEnds.fa.gz
# 400901385 bases (5941742 N's 394959643 real 255835696 upper 139123947 lower)
# in 833173 sequences in 1 files

    #	identical numbers
    #	you can also carefully check the names:
    zcat all.mfa.gz | grep "^>" | awk -F'|' '{print $4}' \
	| sed -e "s/\.[0-9]$//" | sort > mfa.names
    #	should be the same as:
    zcat cloneEnds.fa.gz | grep "^>" | sed -e "s/>//" | sort > clone.names


    # concatenate the text files, too
    bash
    for F in ncbi/*.txt.gz
do
    zcat "${F}"
    echo "${F}" 1>&2
done | gzip > all.txt.gz

    # generate cloneEndPairs.txt and cloneEndSingles.txt
    zcat all.txt.gz >all.txt
    $HOME/kent/src/hg/utils/cloneEndParse.pl all.txt

    #	Reading in end info
    #	Writing out pair info
    #	Writing out singleton info
    #	302264 pairs and 203094 singles
    #	examined all the clone names and all the bac end names in these two
    #	files and compared with business from all.txt to make sure we properly
    #	classified all of them correctly.  We had 833,173 clone sequences,
    #	and 501,135 bac end names

    #	faSplit does not function correctly if given a .gz source file
    #	AND, we need the unzipped file for sequence loading below
    gunzip cloneEnds.fa.gz
    # split
    mkdir splitdir
    cd splitdir
    faSplit sequence ../cloneEnds.fa 100 cloneEnds
    #	Check to ensure no breakage:
    cat *.fa | faSize stdin
# 400901385 bases (5941742 N's 394959643 real 255835696 upper 139123947 lower)
# in 833173 sequences in 1 files
    #	same numbers as before

    # load sequences
    ssh hgwdev
    mkdir /gbdb/hg19/cloneend
    cd /gbdb/hg19/cloneend
      ln -s /hive/data/genomes/hg19/bed/cloneend/cloneEnds.fa .
    cd /tmp
    hgLoadSeq hg19 /gbdb/hg19/cloneend/cloneEnds.fa
    #  Advisory lock created
    # Creating .tab file
    # Adding /gbdb/hg19/cloneend/cloneEnds.fa
    # 833173 sequences
    # Updating seq table
    # Advisory lock has been released
    # All done

##############################################################################
# BACEND SEQUENCE ALIGNMENTS (DONE - 2009-04-28,05-20 - Hiram)
    mkdir -p /hive/data/genomes/hg19/bed/bacends/run.blat
    cd /hive/data/genomes/hg19/bed/bacends/run.blat
    #	going to run separate runs for the golden path sequence vs. the
    #	randoms, haplotypes, chrUn and chrM
    partitionSequence.pl 5000000 20000 /scratch/data/hg19/hg19.2bit \
	/scratch/data/hg19/chrom.sizes 100 -xdir xdir.sh -lstDir tParts \
	| egrep -v "tParts|random|_hap|chrUn" \
	| sed -e "s/.*2bit://; s/:/./" > hg19.list
    ls -1S /hive/data/genomes/hg19/bed/cloneend/splitdir/cloneEnds*.fa \
	> bacEnds.list

    ssh swarm
    cd /hive/data/genomes/hg19/bed/bacends/run.blat

    cat > template << '_EOF_'
#LOOP
runOne.csh $(file1) $(path2) {check out line+ psl/$(root1)/$(file1).$(root2).psl}
#ENDLOOP
'_EOF_'
    # << happy emacs
    cat > runOne.csh << '_EOF_'
#!/bin/csh -fe

set target = $1
set query = $2
set result = $3
set partSpec = `echo $target | sed -e "s/\./:/"`
set start = `echo $partSpec | sed -e "s/.*://; s/-/ /" | awk '{print $1}'`
set end = `echo $partSpec | sed -e "s/.*://; s/-/ /" | awk '{print $2}'`
set range = `echo $start $end | awk '{print $2-$1}'`
set dir = $result:h
set chr = `echo $target | sed -e "s/\..*//"`
set chrSize = `grep -P "^$chr\t" /scratch/data/hg19/chrom.sizes | cut -f2`
set tmpFile = `echo $result | sed -e "s#psl/$chr/#/scratch/tmp/#; s/.psl//"`

# echo $tmpFile
# echo "chr: $chr $start $end -> size: $chrSize, range: $range"
/bin/echo -e "$start\t$partSpec\t$range\t$chr\t$chrSize" > $tmpFile.lift
/bin/mkdir -p $dir
/cluster/bin/x86_64/blat -ooc=/scratch/data/hg19/11.ooc \
        /scratch/data/hg19/hg19.2bit:$partSpec $query $tmpFile.psl
rm -f $result
liftUp -type=.psl $result $tmpFile.lift error $tmpFile.psl
rm -f $tmpFile.lift $tmpFile.psl
'_EOF_'
    # << happy emacs

    gensub2 hg19.list bacEnds.list template jobList
    para create jobList
# 62034 jobs in batch
    # these jobs run quickly, limit them to 250 at a time
    para try, check, -maxJob=250 push, etc ...
# Completed: 62034 of 62034 jobs
# CPU time in finished jobs:     506023s    8433.72m   140.56h    5.86d  0.016 y
# IO & Wait Time:                175853s    2930.88m    48.85h    2.04d  0.006 y
# Average job time:                  11s       0.18m     0.00h    0.00d
# Longest finished job:             752s      12.53m     0.21h    0.01d
# Submission to last job:          3533s      58.88m     0.98h    0.04d

    #	combine the alignments
    time pslSort dirs raw.psl temp psl/chr*
    #	62034 files in 24 dirs
    #	Got 62034 files 249 files per mid file
    #	real    81m2.820s

    #	-rw-rw-r--  1 13410334441 Apr 29 12:00 raw.psl
    # cleanup
    rmdir temp

    time pslReps -nearTop=0.02 -minCover=0.60 -minAli=0.85 -noIntrons \
                raw.psl  bacEnds.psl /dev/null > pslReps.out 2>&1 &
    #	real    5m55.990s
    #	Processed 106254032 alignments
    #	-rw-rw-r--  1   372734361 Apr 29 12:56 bacEnds.psl


    wc -l bacEnds.psl
    #	2852977 bacEnds.psl

    time pslPairs -tInsert=10000 -minId=0.91 -noBin -min=25000 -max=350000 \
	-slopval=10000 -hardMax=500000 -slop -short -long -orphan \
	-mismatch -verbose bacEnds.psl \
	/cluster/data/hg19/bed/cloneend/cloneEndPairs.txt \
	all_bacends bacEnds
    #	Reading pair file
    #	Reading psl file
    #	Creating Pairs
    #	Writing to files
    #	real    0m18.851s
    #	this creates the files:
    #	-rw-rw-r--  1    21178741 Apr 29 13:00 bacEnds.pairs
    #	-rw-rw-r--  1     5250873 Apr 29 13:00 bacEnds.orphan
    #	-rw-rw-r--  1      738045 Apr 29 13:00 bacEnds.short
    #	-rw-rw-r--  1      463560 Apr 29 13:00 bacEnds.slop
    #	-rw-rw-r--  1      146369 Apr 29 13:00 bacEnds.mismatch
    #	-rw-rw-r--  1        3528 Apr 29 13:00 bacEnds.long

    # filter and sort
    awk '$5 >= 300' bacEnds.pairs | sort -k1,1 -k2,2n > bacEndPairs.bed
    awk '$5 >= 300' bacEnds.slop bacEnds.short bacEnds.long \
	bacEnds.mismatch bacEnds.orphan | sort -k1,1 -k2,2n > bacEndPairsBad.bed

    extractPslLoad -noBin bacEnds.psl bacEndPairs.bed \
	bacEndPairsBad.bed | headRest 2 stdin | sort -k14,14 -k16,16n \
	    > bacEndPairs.load.psl

############################################################################
# BACEND Randoms SEQUENCE ALIGNMENTS (DONE - 2009-04-28,05-20 - Hiram)
    mkdir -p /hive/data/genomes/hg19/bed/bacends/run.randoms
    cd /hive/data/genomes/hg19/bed/bacends/run.randoms
    #	this separate run for the randoms, haplotypes, chrUn and chrM
    partitionSequence.pl 5000000 20000 /scratch/data/hg19/hg19.2bit \
	/scratch/data/hg19/chrom.sizes 100 -xdir xdir.sh -lstDir tParts \
	| egrep "random|_hap|chrUn" \
	| sed -e "s/.*2bit://; s/:/./" > random.list
    cat tParts/*.lst | sed -e "s/.*2bit://; s/:/./" >> random.list

    ls -1S /hive/data/genomes/hg19/bed/cloneend/splitdir/cloneEnds*.fa \
	> bacEnds.list

    ssh swarm
    cd /hive/data/genomes/hg19/bed/bacends/run.randoms
    gensub2 random.list bacEnds.list ../run.blat/template jobList
    # very similar runOne.csh script as above, but it doesn't need to do
    #	the lift
    cat > runOne.csh << '_EOF_'
#!/bin/csh -fe

set target = $1
set query = $2
set result = $3
set partSpec = `echo $target | sed -e "s/\./:/"`
set start = `echo $partSpec | sed -e "s/.*://; s/-/ /" | awk '{print $1}'`
set end = `echo $partSpec | sed -e "s/.*://; s/-/ /" | awk '{print $2}'`
set range = `echo $start $end | awk '{print $2-$1}'`
set dir = $result:h
set chr = `echo $target | sed -e "s/\..*//"`
set chrSize = `grep -P "^$chr\t" /scratch/data/hg19/chrom.sizes | cut -f2`
set tmpFile = `echo $result | sed -e "s#psl/$chr/#/scratch/tmp/#; s/.psl//"`

# echo $tmpFile
# echo "chr: $chr $start $end -> size: $chrSize, range: $range"
/bin/echo -e "$start\t$partSpec\t$range\t$chr\t$chrSize" > $tmpFile.lift
/bin/mkdir -p $dir
/cluster/bin/x86_64/blat -ooc=/scratch/data/hg19/11.ooc \
        /scratch/data/hg19/hg19.2bit:$partSpec $query $tmpFile.psl
rm -f $result
mv $tmpFile.psl $result
echo rm -f $tmpFile.lift
'_EOF_'
    # << happy emacs

    # these jobs run fast, do not let too many of them run
    para -maxJob=100 try...check...push
    para time
# Completed: 6762 of 6762 jobs
# CPU time in finished jobs:      20357s     339.29m     5.65h    0.24d  0.001 y
# IO & Wait Time:                 17839s     297.31m     4.96h    0.21d  0.001 y
# Average job time:                   6s       0.09m     0.00h    0.00d
# Longest finished job:             261s       4.35m     0.07h    0.00d
# Submission to last job:           508s       8.47m     0.14h    0.01d

    time pslSort dirs raw.psl temp psl/chr*
    #	6762 files in 69 dirs
    #	Got 6762 files 82 files per mid file
    #	real    6m37.177s

    #	37044 files in 98 dirs
    #	Got 37044 files 192 files per mid file
    #	real    32m24.804s
    #	-rw-rw-r--    1 6487445210 Feb  2 21:08 raw.psl
    time pslReps -nearTop=0.02 -minCover=0.60 -minAli=0.85 -noIntrons \
                raw.psl randomEnds.psl randomReps.psr > pslReps.out 2>&1 &
    #	real    0m5.761s
    #	Processed 1254273 alignments

    # cleanup
    rmdir temp

    wc -l randomEnds.psl
    #	367567 randomEnds.psl

    time pslPairs -tInsert=10000 -minId=0.91 -noBin -min=25000 -max=350000 \
	-slopval=10000 -hardMax=500000 -slop -short -long -orphan \
	-mismatch -verbose randomEnds.psl \
	/cluster/data/hg19/bed/cloneend/cloneEndPairs.txt \
	all_bacends bacEnds
    #	Reading pair file
    #	Reading psl file
    #	Creating Pairs
    #	Writing to files
    #	real    0m11.221s
    #	this creates the files:
    #	-rw-rw-r--  1         0 Apr 29 14:53 bacEnds.slop
    #	-rw-rw-r--  1         0 Apr 29 14:53 bacEnds.short
    #	-rw-rw-r--  1         0 Apr 29 14:53 bacEnds.mismatch
    #	-rw-rw-r--  1         0 Apr 29 14:53 bacEnds.long
    #	-rw-rw-r--  1    141836 Apr 29 14:53 bacEnds.pairs
    #	-rw-rw-r--  1    649907 Apr 29 14:53 bacEnds.orphan

##############################################################################
# BacEnds track - both results loaded together (DONE - 2009-04-29 - Hiram)
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/bacends
    # filter and sort
    awk '$5 >= 300' run.blat/bacEnds.pairs run.randoms/bacEnds.pairs \
	| sort -k1,1 -k2,2n > bacEndPairs.bed
    awk '$5 >= 300' run.blat/bacEnds.slop run.blat/bacEnds.short \
	run.blat/bacEnds.long run.blat/bacEnds.mismatch \
	run.blat/bacEnds.orphan run.randoms/bacEnds.slop \
	run.randoms/bacEnds.short run.randoms/bacEnds.long \
	run.randoms/bacEnds.mismatch run.randoms/bacEnds.orphan \
	    | sort -k1,1 -k2,2n > bacEndPairsBad.bed

    head -5 run.blat/bacEnds.psl > bacEnds.psl
    headRest 5 run.blat/bacEnds.psl > t.psl
    headRest 5 run.randoms/randomEnds.psl >> t.psl
    sort -k14,14 -k16,16n t.psl >> bacEnds.psl
    extractPslLoad -noBin bacEnds.psl bacEndPairs.bed \
	bacEndPairsBad.bed | headRest 2 stdin | sort -k14,14 -k16,16n \
	    > bacEnds.load.psl


    #	load them into the database
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/bacends
    #	CHECK bacEndPairs.bed ID's to make sure they have no blanks in them
    awk '{print $4}' bacEndPairs.bed | grep " "
    awk '{print $5}' bacEndPairs.bed | sort | uniq -c
    #	result should be the scores, no extraneous strings:
    #	156984 1000
    #	   195 300
    #	   316 375
    #	   297 500
    #	  1476 750
    #	edit the file and fix it if it has a bad name.
    hgLoadBed -notItemRgb hg19 bacEndPairs bacEndPairs.bed \
                 -sqlTable=$HOME/kent/src/hg/lib/bacEndPairs.sql
    #	Loaded 208922 elements of size 11
    # note - this track isn't pushed to RR, just used for assembly QA
    hgLoadBed -notItemRgb hg19 bacEndPairsBad bacEndPairsBad.bed \
                 -sqlTable=$HOME/kent/src/hg/lib/bacEndPairsBad.sql
    #	Loaded 79004 elements of size 11
    #hgLoadPsl hg18 -nobin -table=all_bacends bacEnds.load.psl
    # NOTE: truncates file to 0 if -nobin is used
    hgLoadPsl hg19 -table=all_bacends bacEnds.load.psl
    # one complaint, there appears to be a bogus insert count in one
    #	of the blat results:
# < 585   797     67      0       3       2       -63     9       79188   +      AQ743980 852     42      846     chr19_gl000208_random   92689   4045    84100  11       14,124,84,496,53,6,20,28,28,10,4,       42,56,180,200,696,750,756,776,804,832,842,      4045,5767,7086,83449,83946,83999,84006,84027,84056,84085,84096,
Became:
# > 585   797     67      0       3       2       0       9       79188   +	 AQ743980 852     42      846     chr19_gl000208_random   92689   4045	84100  11       14,124,84,496,53,6,20,28,28,10,4,	42,56,180,200,696,750,756,776,804,832,842,	4045,5767,7086,83449,83946,83999,84006,84027,84056,84085,84096,

    hgsql -N -e "select count(*) from all_bacends;" hg19
    #	 2289275
    hgsql -N -e "select count(*) from all_bacends;" hg18
    #	1727387
    hgsql -N -e "select count(*) from all_bacends;" hg17
    #	 1729146

    nice featureBits hg19 all_bacends
# 230917362 bases of 2897316137 (7.970%) in intersection
    nice featureBits hg18 all_bacends
# 227770876 bases of 2881515245 (7.905%) in intersectio
    nice featureBits hg17 all_bacends
# 225763317 bases of 2866216770 (7.877%) in intersection

    nice featureBits hg19 bacEndPairs
# 236889607 bases of 2897316137 (8.176%) in intersection
    nice featureBits hg18 bacEndPairs
# 162690030 bases of 2881515245 (5.646%) in intersection
    nice featureBits hg17 bacEndPairs
# 162099487 bases of 2866216770 (5.656%) in intersection

    nice featureBits hg19 bacEndPairsBad
# 38344094 bases of 2897316137 (1.323%) in intersection
    nice featureBits hg18 bacEndPairsBad
# 37326990 bases of 2881515245 (1.295%) in intersection
    nice featureBits hg17 bacEndPairsBad
# 37437558 bases of 2866216770 (1.306%) in intersection

############################################################################
# STS MARKERS (DONE - 2009-04-30 - 2009-05-06 - Hiram)
    mkdir /hive/data/outside/ncbi/sts.2009-04
    cd /hive/data/outside/ncbi
    ln -s sts.2009-04 sts.11
    cd /hive/data/outside/ncbi/sts.2009-04
    wget --timestamping ftp://ftp.ncbi.nih.gov/repository/UniSTS/UniSTS.sts
    wget --timestamping ftp://ftp.ncbi.nih.gov/repository/UniSTS/UniSTS.aliases
    wget --timestamping ftp://ftp.ncbi.nih.gov/blast/db/FASTA/sts.gz
    gunzip sts.gz
    mv sts dbSTS.fa

    #	these items are copied in from the previous builds
    cp -p /cluster/data/ncbi/sts.10/all.STS.fa ./all.STS.fa.prev
    cp -p /cluster/data/ncbi/sts.10/stsInfo2.bed ./stsInfo2.bed.prev
    #	edit stsInfo2.bed.prev for a
    #	manual fixup of error that is in the hg18 bed file, replace
    #	the line for AFM067XA9 to fix bogus long list of aliases to be:
# 22788^IAFM067XA9^I1^IZ66598^I1^IGDB:1221611,^I5^I067XA9,GDB:1221611,W202,Z66598,SWSS2303^I69047^I0^I^ITCTTGGGGTTTAATTGCTTT^ICTTTGCCACAATCTTACACA^I149^IHomo sapiens^I1^I2^I6453,6454,^I0^I^I^I^I0^I0^I^I^I0^I0^IAFM067XA9^Ichr7^I145^I0^I^I^I0^I0^I^I^I0^I0^I^I^I0^I0^I^I^I0^I0^I^I^I0^I0
    #	as taken directly out of the hg18.stsInfo2 table which was fixed
    #	by Bob and Archana

    # Convert the title line of the dbSTS.fa file
    #	Verify that column 3 only contains gb emb dbj
    grep "^>" dbSTS.fa | awk -F'|' '{print $3}' | sort | uniq -c 
#   39124 dbj
#   57375 emb
# 1212541 gb
    #	if that is true, this sed will work:
    cat dbSTS.fa \
	| sed -e "s#^>gi.[0-9]*.gb.#>#; s#^>gi.[0-9]*.emb.#>#; s#^>gi.[0-9]*.dbj.#>#; s#\.[0-9]|.*##" \
	    > UniSTS.convert.fa

    # get accessions
    grep ">" UniSTS.convert.fa | sed -e "s/^>//" | sort > UniSTS.acc
    #	head and tail that to ensure names are reasonable, odd names would
    #	show up at the beginning or end
    wc -l UniSTS.acc
    #	1309040 UniSTS.acc

    # NOTE: updateStsInfo creates new stsInfo2.bed, all.primers,
    #   all.STS.fa, stsAlias.bed files

    updateStsInfo -verbose=1 -gb=UniSTS.acc stsInfo2.bed.prev all.STS.fa.prev \
	UniSTS.sts UniSTS.aliases UniSTS.convert.fa new

    #	verify the number of aliases is reasonable:
    awk '{print $3}' new.alias | sort | uniq -c | sort -rn | less
    #	50 D7S831
    #	34 CHLC.GATA2B06.465
    #	24 CHLC.GATA11E11
    #	23 AFM276ZF5
    #	23 AFM273YH9
    #	22 SHGC-133043
    #	... etc ...
    #	verify there are no unusually long or short lines:
    awk '{printf "%d\n", length($0)}' new.info | sort -n | head -3
    #	143
    #	144
    #	144
    awk '{printf "%d\n", length($0)}' new.info | sort -n | tail -3
    #	552
    #	553
    #	644
    # check for null in the new files:
    grep -i null new.*
    #	if the new files look good, they can become the set to use:
    mv new.info stsInfo2.bed
    mv new.primers all.primers
    mv new.alias stsAlias.bed
    mv new.fa all.STS.fa

    # get list of all STS id's in the fasta file
    sed -n 's/^>\([0-9][0-9]*\) .*/\1/p' all.STS.fa | sort -n >  all.STS.id
    wc -l all.STS.id
    # 100520 total sequences
    # in hg18 this was: 93698 total sequences
    $HOME/kent/src/hg/stsMarkers/convertPrimerToFA all.primers > all.primers.fa
    # check that fasta file for unusual length sequences:
    faSize all.primers.fa
# 97815329 bases (83677626 N's 14137703 real 14137703 upper 0 lower) in 317592 sequences in 1 files
# Total size: mean 308.0 sd 279.3 min 40 (dbSTS_144) max 30000 (dbSTS_156892) median 244

    # Copy stsInfo2.bed and stsAlias.bed to data directory becuase
    # these will be loaded into the database later
    mkdir -p /hive/data/genomes/hg19/bed/sts
    cp -p stsInfo2.bed /hive/data/genomes/hg19/bed/sts/
    cp -p stsAlias.bed /hive/data/genomes/hg19/bed/sts/

    # Create sts sequence alignments
    mkdir /hive/data/genomes/hg19/bed/sts/split

    faSplit sequence all.STS.fa 100 /hive/data/genomes/hg19/bed/sts/split/sts

    ssh swarm
    mkdir /hive/data/genomes/hg19/bed/sts/run
    cd /hive/data/genomes/hg19/bed/sts/run

    #	going to run separate runs for the golden path sequence vs. the
    #	randoms, haplotypes, chrUn and chrM
    #	40,000,000 chunck sizes, 20,000 overlap
    partitionSequence.pl 40000000 20000 /scratch/data/hg19/hg19.2bit \
	/scratch/data/hg19/chrom.sizes 100 -lstDir tParts \
	| egrep -v "tParts|random|_hap|chrUn" \
	| sed -e "s/.*2bit://;" > hg19.list
    ls -1S ../split > sts.list

    cat > template << '_EOF_'
#LOOP
runOne.csh $(file1) $(root2) {check out line+ psl/$(file1)/$(root2).psl}
#ENDLOOP
'_EOF_'
    # << happy emacs

    cat > runOne.csh << '_EOF_'
#!/bin/csh -fe

set partSpec = $1
set query = $2.fa
set result = $3
set tmpFile = "/scratch/tmp/$1.$2"
set start = `echo $partSpec | sed -e "s/.*://; s/-/ /" | awk '{print $1}'`
set end = `echo $partSpec | sed -e "s/.*://; s/-/ /" | awk '{print $2}'`
set range = `echo $start $end | awk '{print $2-$1}'`
set chr = `echo $partSpec | sed -e "s/:.*//"`
set chrSize = `grep -P "^$chr\t" /scratch/data/hg19/chrom.sizes | cut -f2`
/bin/echo -e "$start\t$partSpec\t$range\t$chr\t$chrSize" > $tmpFile.lift
/bin/mkdir -p psl/$partSpec
/bin/rm -f $tmpFile
/cluster/bin/x86_64/blat -ooc=/scratch/data/hg19/11.ooc \
    /scratch/data/hg19/hg19.2bit:$partSpec \
	../split/${query} -stepSize=5 $tmpFile.psl
/bin/rm -f $result
/cluster/bin/x86_64/liftUp -type=.psl $result $tmpFile.lift error $tmpFile.psl
# rm -f $tmpFile.lift $tmpFile.psl
'_EOF_'
    # << happy emacs
    chmod +x runOne.csh

    gensub2 hg19.list sts.list template jobList
    #	these jobs run quickly, allow only 100 at a time
    para -maxJob=100 create jobList
# 8367 jobs in batch
    para try ... check ... push ... etc
# Completed: 8366 of 8366 jobs
# CPU time in finished jobs:      89744s    1495.74m    24.93h    1.04d  0.003 y
# IO & Wait Time:                 25467s     424.44m     7.07h    0.29d  0.001 y
# Average job time:                  14s       0.23m     0.00h    0.00d
# Longest finished job:              53s       0.88m     0.01h    0.00d
# Submission to last job:          1592s      26.53m     0.44h    0.02d

    #	and, run the randoms as a separate run:
    mkdir /hive/data/genomes/hg19/bed/sts/run.randoms
    cd /hive/data/genomes/hg19/bed/sts/run.randoms
    partitionSequence.pl 40000000 20000 /scratch/data/hg19/hg19.2bit \
	/scratch/data/hg19/chrom.sizes 100 -lstDir tParts \
	| egrep "tParts|random|_hap|chrUn"
    cat tParts/* | sed -e "s/.*2bit://;" > hg19.list
    ls -1S ../split > sts.list
    cat > template << '_EOF_'
#LOOP
runOne.csh $(file1) $(root2) {check out line+ psl/$(file1)/$(root2).psl}
#ENDLOOP
'_EOF_'
    # << happy emacs

    cat > runOne.csh << '_EOF_'
#!/bin/csh -fe

set partSpec = $1
set query = $2.fa
set result = $3
set tmpFile = "/scratch/tmp/$1.$2"
/bin/mkdir -p psl/$partSpec
/bin/rm -f $tmpFile
/cluster/bin/x86_64/blat -ooc=/scratch/data/hg19/11.ooc \
    /scratch/data/hg19/hg19.2bit:$partSpec \
	../split/${query} -stepSize=5 $tmpFile.psl
/bin/rm -f $result
mv $tmpFile.psl $result
/bin/rm -f $tmpFile.psl
'_EOF_'
    # << happy emacs
    chmod +x runOne.csh

    gensub2 hg19.list sts.list template jobList
    #	these jobs run quickly, allow only 100 at a time
    para -maxJob=100 create jobList
# 6486 jobs in batch
    para try ... check ... push ... etc
# Completed: 6486 of 6486 jobs
# CPU time in finished jobs:       2206s      36.77m     0.61h    0.03d  0.000 y
# IO & Wait Time:                 16505s     275.08m     4.58h    0.19d  0.001 y
# Average job time:                   3s       0.05m     0.00h    0.00d
# Longest finished job:              21s       0.35m     0.01h    0.00d
# Submission to last job:           601s      10.02m     0.17h    0.01d

    # Compile sts sequence results
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/sts/run
    time pslSort dirs raw.psl temp psl/chr*
    #	8366 files in 89 dirs
    #	Got 8366 files 91 files per mid file
    #	real    8m50.714s
    #	-rw-rw-r--  1 810438277 May  1 11:45 raw.psl
    cd /hive/data/genomes/hg19/bed/sts/run.randoms
    time pslSort dirs raw.psl temp psl/chr*
    #	6486 files in 69 dirs
    #	Got 6486 files 81 files per mid file
    #	real    1m42.120s
    #	-rw-rw-r--  1 18378188 May  1 11:52 raw.psl

    rmdir temp
    cd /hive/data/genomes/hg19/bed/sts
    cat run*/raw.psl | egrep -v "^$|^psLayout|^match|^ |^-" \
	| pslReps -nearTop=0.0001 -minCover=0.6 -minAli=0.8 -noIntrons stdin \
	stsMarkers.psl /dev/null
    #	Processed 7412166 alignments
    #	-rw-rw-r-- 1 12031760 May  1 11:57 stsMarkers.psl

    $HOME/kent/src/hg/stsMarkers/extractPslInfo -h stsMarkers.psl
    # creates stsMarkers.psl.initial
    #	-rw-rw-r-- 1  4485053 May  1 12:06 stsMarkers.psl.initial
    wc -l stsMarkers.psl.initial
    #	101338  stsMarkers.psl.initial
    #	this command needs a chrom_names file to work correctly with this
    #	new style of layout for hg19:
    cd /hive/data/genomes/hg19
    cut -f1 chrom.sizes | sed -e "s/chr//" > chrom_names
    cd /hive/data/genomes/hg19/bed/sts

    $HOME/kent/src/hg/stsMarkers/findAccession.pl -agp stsMarkers.psl.initial \
	/cluster/data/hg19
    wc -l stsMarkers.psl.initial.acc
    #	101338  stsMarkers.psl.initial.acc

    sort -k4,4n stsMarkers.psl.initial.acc > stsMarkers.final

    # determine found markers (4th field in file)
    cut -f 4 stsMarkers.final | sort -n -u > stsMarkers.found
    wc -l stsMarkers.found
    #	96472 stsMarkers.found
    #	out of 100520 total sequences from:
    wc -l /hive/data/outside/ncbi/sts.2009-04/all.STS.id
    #	There are lots of duplicates:
    wc -l stsMarkers.final
    #	101338 stsMarkers.final
    #	And a lot of them are just completely haywire:
    awk '$3-$2 < 1001' stsMarkers.final | wc -l
    #	98382
    #	filter out markers that are too long
    awk '$3-$2 < 1001' stsMarkers.final > stsMarkers.1K.size.filtered

    #  alignment of primers
    ssh swarm
    cd /hive/data/outside/ncbi/sts.2009-04
    awk '$0 !~ /[^ACGT0-9\-\t]/ && (length($2) > 10) && (length($3) > 10) {printf "dbSTS_%s\t%s\t%s\n", $1,$2,$3}' \
	    all.primers > all.primers.ispcr
    mkdir primerAlign
    cd primerAlign
    mkdir split
    cd split
    split -l 5000 ../../all.primers.ispcr primer_
    ls > ../primer.list

    cd ..
    #	we need a 10.ooc file for this business
    time blat /scratch/data/hg19/hg19.2bit \
	/dev/null /dev/null -tileSize=10 -makeOoc=10.ooc -repMatch=1024
# Wrote 146902 overused 10-mers to 10.ooc
# real    19m16.758s

    # separate runs for whole genome vs. randoms
    mkdir run
    cd run
    partitionSequence.pl 40000000 20000 /scratch/data/hg19/hg19.2bit \
	/scratch/data/hg19/chrom.sizes 100 -lstDir tParts \
	| egrep -v "tParts|random|_hap|chrUn" \
	| sed -e "s/.*2bit://;" > hg19.list
    cat > runOne.csh << '_EOF_'
#!/bin/csh -fe

set partSpec = $1
set primer = ../split/$2
set result = $3
set tmpFile = "/scratch/tmp/$1.$2"
set start = `echo $partSpec | sed -e "s/.*://; s/-/ /" | awk '{print $1}'`
set end = `echo $partSpec | sed -e "s/.*://; s/-/ /" | awk '{print $2}'`
set range = `echo $start $end | awk '{print $2-$1}'`
set chr = `echo $partSpec | sed -e "s/:.*//"`
set chrSize = `grep -P "^$chr\t" /scratch/data/hg19/chrom.sizes | cut -f2`
/bin/echo -e "$start\t$partSpec\t$range\t$chr\t$chrSize" > $tmpFile.lift
/bin/mkdir -p psl/$partSpec
/bin/rm -f $tmpFile.psl
/cluster/bin/x86_64/isPcr -out=psl -minPerfect=2 -maxSize=5000 -tileSize=10 \
    -ooc=/hive/data/outside/ncbi/sts.2009-04/primerAlign/10.ooc -stepSize=5 \
	/scratch/data/hg19/hg19.2bit:$partSpec $primer $tmpFile.psl
/bin/rm -f $result
/cluster/bin/x86_64/liftUp -type=.psl $result $tmpFile.lift error $tmpFile.psl
rm -f $tmpFile.lift $tmpFile.psl
'_EOF_'
    # << happy emacs
    chmod +x runOne.csh

    cat > template << '_EOF_'
#LOOP
runOne.csh $(file1) $(root2) {check out line+ psl/$(file1)/$(root2).psl}
#ENDLOOP
'_EOF_'
    # << happy emacs

    gensub2 hg19.list ../primer.list template jobList
    para create jobList
# 5696 jobs in batch
    para try ... check ... push ... etc
# Completed: 5696 of 5696 jobs
# CPU time in finished jobs:     203899s    3398.32m    56.64h    2.36d  0.006 y
# IO & Wait Time:                 22049s     367.48m     6.12h    0.26d  0.001 y
# Average job time:                  40s       0.66m     0.01h    0.00d
# Longest finished job:            5314s      88.57m     1.48h    0.06d
# Submission to last job:          5418s      90.30m     1.50h    0.06d
# Estimated complete:                 0s       0.00m     0.00h    0.00d

    #	sort and filter the results
    cd psl
    pslSort dirs raw.psl temp chr*
    #	5696 files in 89 dirs
    #	Got 5696 files 75 files per mid file
    #	-rw-rw-r-- 1 456802973 May  4 13:32 raw.psl
    cd ..
    mkdir filter
    pslQuickFilter -minMatch=26 -maxMismatch=5 \
        -maxTinsert=5000 -verbose psl/ filter/
    #	-rw-rw-r-- 1 50302564 May  4 13:35 raw.psl

    #	And, for the randoms
    mkdir /hive/data/outside/ncbi/sts.2009-04/primerAlign/runRandoms
    cd /hive/data/outside/ncbi/sts.2009-04/primerAlign/runRandoms
    
    partitionSequence.pl 40000000 20000 /scratch/data/hg19/hg19.2bit \
	/scratch/data/hg19/chrom.sizes 100 -lstDir tParts \
	| egrep "tParts|random|_hap|chrUn" \
	| sed -e "s/.*2bit://;" > hg19.list
    cat tParts/* | sed -e "s/.*2bit://;" > hg19.list
    cat tParts/* > hg19.list

    cat > runOne.csh << '_EOF_'
#!/bin/csh -fe

set partSpec = $1
set primer = ../split/$2
set result = $3
set tmpFile = "/scratch/tmp/$1.$2"
/bin/mkdir -p psl/$partSpec
/bin/rm -f $tmpFile.psl
/cluster/bin/x86_64/isPcr -out=psl -minPerfect=2 -maxSize=5000 -tileSize=10 \
    -ooc=/hive/data/outside/ncbi/sts.2009-04/primerAlign/10.ooc -stepSize=5 \
	/scratch/data/hg19/hg19.2bit:$partSpec $primer $tmpFile.psl
/bin/rm -f $result
mv $tmpFile.psl $result
'_EOF_'
    # << happy emacs
    chmod +x runOne.csh

    #	can not use line+ check here, many of them are empty
    cat > template << '_EOF_'
#LOOP
runOne.csh $(file1) $(root2) {check out line psl/$(file1)/$(root2).psl}
#ENDLOOP
'_EOF_'
    # << happy emacs

    gensub2 hg19.list ../primer.list template jobList
    #	they run quickly, limit to 100
    para -maxJob=100 create jobList
    para try ... check ... push ... etc
# Completed: 4416 of 4416 jobs
# CPU time in finished jobs:       1746s      29.09m     0.48h    0.02d  0.000 y
# IO & Wait Time:                 11407s     190.12m     3.17h    0.13d  0.000 y
# Average job time:                   3s       0.05m     0.00h    0.00d
# Longest finished job:               8s       0.13m     0.00h    0.00d
# Submission to last job:           147s       2.45m     0.04h    0.00d

    #	sort and filter the results
    cd psl
    pslSort dirs raw.psl temp chr*
    #	4416 files in 69 dirs
    #	Got 4416 files 66 files per mid file
    rmdir temp
    #	-rw-rw-r-- 1 9066053 May  4 13:31 raw.psl

    #	putting the two runs together
    mkdir /hive/data/outside/ncbi/sts.2009-04/primerAlign/psl
    cd /hive/data/outside/ncbi/sts.2009-04/primerAlign/psl
    ln -s ../run/filter/raw.psl run.psl
    ln -s ../runRandoms/filter/raw.psl runRandoms.psl
    #	-rw-rw-r-- 1 50302564 May  4 13:35 run.psl
    #	-rw-rw-r-- 1   825973 May  4 13:35 runRandoms.psl
    cd ..
    pslSort dirs primers.psl temp psl
    #	2 files in 1 dirs
    #	Got 2 files 1 files per mid file
    #	-rw-rw-r-- 1 51128110 May  4 13:39 primers.psl
    wc -l primers.psl
    #	448107 primers.psl
    rmdir temp
    pslFilterPrimers primers.psl ../all.primers primers.filter.psl
    # creates primers.filter.unlifted.psl.notfound.primers
    wc -l primers*
    #	237962 primers.filter.psl
    #	97191 primers.filter.psl.notfound.primers

    #	see if ePCR can find some of these notfound
    ssh swarm
    mkdir /hive/data/outside/ncbi/sts.2009-04/primerAlign/epcr
    cd /hive/data/outside/ncbi/sts.2009-04/primerAlign/epcr

    mkdir split
    cd split
    split -l 5000 ../../primers.filter.psl.notfound.primers  primers_
    cd ..
    ls -1S split > primers.lst
    partitionSequence.pl 40000000 20000 /scratch/data/hg19/hg19.2bit \
	/scratch/data/hg19/chrom.sizes 100 -lstDir tParts \
	| grep -v tParts | sed -e "s/.*2bit://;" > hg19.list
    cat tParts/* | sed -e "s/.*2bit://;" >> hg19.list

    cat > runOne.csh << '_EOF_'
#!/bin/csh -fe

set partSpec = $1
set primer = split/$2
set result = $3
set tmpFile = "/scratch/tmp/$1.$2"
set start = `echo $partSpec | sed -e "s/.*://; s/-/ /" | awk '{print $1}'`
set end = `echo $partSpec | sed -e "s/.*://; s/-/ /" | awk '{print $2}'`
set range = `echo $start $end | awk '{print $2-$1}'`
set chr = `echo $partSpec | sed -e "s/:.*//"`
set chrSize = `grep -P "^$chr\t" /scratch/data/hg19/chrom.sizes | cut -f2`
/bin/echo -e "$start\t$partSpec\t$range\t$chr\t$chrSize" > $tmpFile.lift
/bin/mkdir -p epcr/$partSpec
/bin/rm -f $tmpFile.psl
twoBitToFa /scratch/data/hg19/hg19.2bit:$partSpec $tmpFile.fa
/cluster/bin/scripts/runEpcr64 $primer $tmpFile.fa $tmpFile.epcr
/bin/rm -f $result
/bin/mv $tmpFile.epcr $result
rm -f $tmpFile.fa $tmpFile.lift $tmpFile.psl $tmpFile.*
'_EOF_'
    # << happy emacs
    chmod +x runOne.csh

    cat > template << '_EOF_'
#LOOP
runOne.csh $(file1) $(root2) {check out line epcr/$(file1)/$(root2).epcr}
#ENDLOOP
'_EOF_'
    # << happy emacs

    gensub2 hg19.list primers.lst template jobList
    para create jobList
	# 3160 jobs
    para try ... check ... push ... etc ...
# Completed: 3160 of 3160 jobs
# CPU time in finished jobs:      86253s    1437.54m    23.96h    1.00d  0.003 y
# IO & Wait Time:                 11196s     186.61m     3.11h    0.13d  0.000 y
# Average job time:                  31s       0.51m     0.01h    0.00d
# Longest finished job:              89s       1.48m     0.02h    0.00d
# Submission to last job:           237s       3.95m     0.07h    0.00d

    find ./epcr -type f | xargs cat > all.epcr
    wc -l all.epcr
    #	797286 all.epcr
    # convert the coordinates from the partitionSequence.pl to a lift file
    awk '{print $1}' all.epcr | sort -u > hg19.partSpec.txt
    $HOME/kent/src/hg/stsMarkers/liftFromSpec.pl hg19 hg19.partSpec.txt \
	> all.epcr.lift
    cat all.epcr | sed -e "s/\.\./ /; s/  */\t/g" \
	| liftUp -type=.bed stdout all.epcr.lift error stdin \
	| awk '
{
printf "%s %d..%d %d %d\n", $1, $2, $3, $4, $5
}
' > all.epcr.lifted

    pslFilterPrimers -epcr=all.epcr.lifted -verbose=1 ../primers.psl \
    /cluster/home/hiram/bin/x86_64/pslFilterPrimers -epcr=all.epcr.lifted \
	-verbose=1 ../primers.psl ../../all.primers epcr.primers.psl
    #	this took a long time, many hours
# -rw-rw-r--   1  2785254 May  5 17:28 epcr.not.found
# -rw-rw-r--   1 27343510 May  5 17:28 epcr.primers.psl
# -rw-rw-r--   1  1616885 May  5 17:28 epcr.primers.psl.notfound.primers

    time ./epcrToHgPsl.pl epcr.not.found ../../all.primers \
    time $HOME/kent/src/hg/stsMarkers/epcrToPsl epcr.not.found \
	../../all.primers /hive/data/genomes/hg19
    #	real    69m38.444s
    #	-rw-rw-r--   1        0 May  6 14:18 epcr.not.found.nomatch
    #	-rw-rw-r--   1  8369138 May  6 15:26 epcr.not.found.psl

    #	combining everything together now
    cd /hive/data/outside/ncbi/sts.2009-04/primerAlign

    sort -u primers.filter.psl epcr/epcr.primers.psl epcr/epcr.not.found.psl \
                | sort -k15,15 -k17,17n > primers.final.psl
    wc -l primers.final.psl
    #	310705 primers.final.psl

    time $HOME/kent/src/hg/stsMarkers/fixPrimersQueryGaps.pl \
        ../all.primers primers.final.psl > primers.final.fix.psl
    #	real    0m19.580s
    wc -l primers.final.fix.psl
    #	310705 primers.final.fix.psl

    # Extract relevant info, make alignments unique, and create final file to
    #	be merged with full sequence alignments
    $HOME/kent/src/hg/stsMarkers/extractPslInfo -h primers.final.fix.psl
    #	real    0m15.303s
    #	-rw-rw-r-- 1 15660447 May  6 15:44 primers.final.fix.psl.initial
    wc -l primers.final.fix.psl.initial
    #	308210 primers.final.fix.psl.initial
    $HOME/kent/src/hg/stsMarkers/findAccession.pl -agp \
	primers.final.fix.psl.initial /hive/data/genomes/hg19
    wc -l primers.final.fix.psl.initial.acc
    #	308210 primers.final.fix.psl.initial.acc

    $HOME/kent/src/hg/stsMarkers/getStsId ../stsInfo2.bed \
	primers.final.fix.psl.initial.acc | sort -k 4n > primers.final
    wc -l primers.final
    # 308210 primers.final
    #	There doesn't appear to be any use for this primers.ids list
    #	except for curiosity.  Check the head and tail of this list to
    #	verify no garbage is in here.  There should just be numbers.
    awk '{print $4}' primers.final | sort -n | uniq > primers.ids
    wc -l primers.ids
    #	290961 primers.ids

    # Merge primer and sequence files to create final bed file
    # Merge (combineSeqPrimerPos) takes about an hour to run
    cd /hive/data/genomes/hg19/bed/sts
    time $HOME/kent/src/hg/stsMarkers/combineSeqPrimerPos stsMarkers.final \
	/hive/data/outside/ncbi/sts.2009-04/primerAlign/primers.final
    #	real    0m12.310s
    #	-rw-rw-r-- 1 15222346 May  6 15:55 stsMarkers_pos.rdb
    wc -l stsMarkers_pos.rdb
    #	315308 stsMarkers_pos.rdb

    time /cluster/bin/scripts/createSTSbed \
	/hive/data/outside/ncbi/sts.2009-04/stsInfo2.bed  \
	stsMarkers_pos.rdb > stsMap.bed
    #	real    0m31.886s
    #	-rw-rw-r-- 1 38244880 May  6 16:25 stsMap.bed
    wc -l stsMap.bed
    #	305914 stsMap.bed

    # Set up sequence files
    ssh hgwdev
    mkdir /gbdb/hg19/sts.11/
    ln -s /hive/data/outside/ncbi/sts.11/all.STS.fa \
	/gbdb/hg19/sts.11/all.STS.fa
    ln -s /hive/data/outside/ncbi/sts.11/all.primers.fa \
        /gbdb/hg19/sts.11/all.primers.fa

    # Load all files
    cd /hive/data/genomes/hg19/bed/sts
    hgLoadSeq hg19 /gbdb/hg19/sts.11/all.STS.fa /gbdb/hg19/sts.11/all.primers.fa
    #	Creating seq.tab file
    #	Adding /gbdb/hg19/sts.11/all.STS.fa
    #	100520 sequences
    #	Adding /gbdb/hg19/sts.11/all.primers.fa
    #	317592 sequences
    #	Updating seq table
    #	Advisory lock has been released
    #	All done


    hgsql hg19 < $HOME/kent/src/hg/lib/stsInfo2.sql
    hgsql hg19 < $HOME/kent/src/hg/lib/stsAlias.sql
    #	these files already exist here from previous operations
    # cp -p /hive/data/outside/ncbi/sts.11/{stsInfo2.bed,stsAlias.bed} .
    hgsql hg19 -e 'load data local infile "stsInfo2.bed" into table stsInfo2'
    hgsql hg19 -e 'load data local infile "stsAlias.bed" into table stsAlias'
    #	a couple minutes for each load above
    #	filter the stsMap.bed to eliminate items longer than 5,000 bases,
    #	takes out about 850:
    awk '$3-$2 < 5001' stsMap.bed | sort -k1,1 -k2,2n \
	> stsMap.filtered.5000.bed

    hgLoadBed -notItemRgb -noBin -tab \
	-sqlTable=$HOME/kent/src/hg/lib/stsMap.sql hg19 stsMap \
	    stsMap.filtered.5000.bed
    #	Loaded 305064 elements of size 28

    ln -s \
/hive/data/outside/ncbi/sts.2009-04/primerAlign/primers.final.fix.psl \
	primers.psl

    hgLoadPsl -nobin -table=all_sts_primer hg19 primers.psl
    hgLoadPsl -nobin -table=all_sts_seq hg19 stsMarkers.psl

##############################################################################
# FISH CLONES (WORKING - 2009-04-29 - Hiram)
# The STS Marker and BAC End Pairs tracks must be completed prior to
# creating this track.  

    mkdir /hive/data/outside/ncbi/fishClones/fishClones.2009-04/
    cd /hive/data/outside/ncbi/fishClones/fishClones.2009-04/

# Download information from NCBI
        # point browser at:
#   http://www.ncbi.nlm.nih.gov/genome/cyto/cytobac.cgi?CHR=all&VERBOSE=ctg
# change "Sequence tag:" to "placed on contig"
        # change "Show details on sequence-tag" to "yes"
        # change "Download or Display" to "Download table for UNIX"
        # press Submit - save as
# /hive/data/outside/ncbi/fishClones/fishClones.2009-04/hbrc.txt
    chmod 664 /hive/data/outside/ncbi/fishClones/fishClones.2009-04/hbrc.txt

#	Unfortunately the format of this hbrc file has changed since
#	last time.  The columns have been rearranged, and one important
#	column is missing, the contig information.  So, let's see if we
#	can recover the original format by putting this together with
#	some other things we have here.
    $HOME/kent/src/hg/fishClones/fixup.hbrc.pl hbrc.txt \
	/hive/data/genomes/hg19/bed/fishClones/seq_clone.pmd > fixed.hbrc.txt \
	    2> dbg
XXX - need to get this seq_clone.pmd from NCBI, maybe Paul Kitts
    #	the seq_clone.pmd file was obtained via email from Wonhee Jang
    #	jang at ncbi.nlm.nih.gov - I have asked for clarification where
    #	such a file can be fetched without resorting to email.

# Get current clone/accession information
    wget --timestamping ftp://ftp.ncbi.nih.gov/repository/clone/reports/clac.out
http://www.ncbi.nlm.nih.gov/genome/clone/DATA/clac.out

# Create initial Fish Clones bed file
    ssh kkstore02
    mkdir /hive/data/genomes/hg19/bed/fishClones
    cd /hive/data/genomes/hg19/bed/fishClones

    # Copy previous sts info from fhcrc
    cp -p /hive/data/genomes/hg18/bed/fishClones/fhcrc.sts .
    #	This fhcrc.sts listing doesn't change.  It is merely a listing
    #	of aliases that remain in effect.

    #	Create cl_acc_gi_len file form cloneend information:
    grep -v "^#" /hive/data/genomes/hg19/bed/cloneend/all.txt \
    | awk '{gsub(".[0-9]*$", "", $2);
	printf "%s\t%s\t%s\t%s\t%s\t%s\n", $1,$2,$3,$4,$5,$8}' > cl_acc_gi_len

    hgsql -N \
	-e "select chrom,chromStart,chromEnd,contig from ctgPos;" hg19 \
	| sort -k1,1 -k2,2n > ctgPos.bed
    hgsql -N \
-e "select chrom,chromStart,chromEnd,frag,0,strand from gold;" hg19 \
	| sort -k1,1 -k2,2n > gold.bed
    hgsql -N \
-e "select tName,tStart,tEnd,qName,0,strand from all_bacends;" hg19 \
	| sort -k1,1 -k2,2n > all_bacends.bed
    hgsql -N \
-e "select chrom,chromStart,chromEnd,name,score,strand from bacEndPairs;" hg19 \
	| sort -k1,1 -k2,2n > bacEndPairs.bed



    ssh hgwdev
    #	have to be on hgwdev for this since it is going to read from the
    #	database.  Had to work on this program to get it past what is
    #	evidently a bad entry in hbrc.fixed where columns of information
    #	are missing for one clone in particular
    time fishClones -verbose=2 -fhcrc=fhcrc.sts -noBin hg19 \
	/hive/data/genomes/hg19/bed/ncbiCytoBand/contig/fixed.hbrc.txt \
	/hive/data/outside/ncbi/fishClones/fishClones.2009-04/clac.out \
         ./cl_acc_gi_len \
	/hive/data/genomes/hg19/bed/bacends/bacEnds.load.psl \
            fishClones
    #	real    2m4.708s
# Reading Fish Clones file /hive/data/genomes/ncbi/fishClones/fishClones.2006-01/hbrc.fixed
# reading fishInfo file /hive/data/genomes/ncbi/fishClones/fishClones.2006-01/fixed.hbrc.txt
# Reading Clone/Acc (clac.out) file /hive/data/genomes/ncbi/fishClones/fishClones.2006-01/clac.out
# Reading BAC Ends file ./cl_acc_gi_len
# Reading BAC Ends psl file /hive/data/genomes/hg19/bed/bacends/bacEnds.lifted.psl
# Reading additional STS Marker links fhcrc.sts
# Determining good positions
#	findClonePos: determining positions of fish clones
# Writing output file
# ERROR: at line # 170, no cytoband info for chrX:104048913-104206974
# RP11-79L11
# ERROR: at line # 171, no cytoband info for chrX:104048913-104206974
# RP11-79L11

    # Load the track
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/fishClones
    hgLoadBed -notItemRgb -noBin -tab \
        -sqlTable=$HOME/kent/src/hg/lib/fishClones.sql \
	hg19 fishClones fishClones.bed
    #	Loaded 9461 elements of size 16

##############################################################################
# CytoBands from Wonhee Jang at NCBI (DONE - 2009-06-10 - Hiram)

    mkdir /hive/data/genomes/hg19/bed/ncbiCytoBand
    cd /hive/data/genomes/hg19/bed/ncbiCytoBand
    #	received the following files via email:
    ls -ogrt
# -rw-rw-r-- 1 187930 Jun 10 13:53 ideogram
# -rw-rw-r-- 1 672327 Jun  8 09:55 fish.markers.bed

    #	created cytobands.bed from the ideogram file with:
    cat << '_EOF_' > ideoToCytoBand.pl
#!/usr/bin/env perl

use strict;
use warnings;

open (FH,"<ideogram") or die "can not read ideogram";

while (my $line = <FH>) {
    next if $line =~ m/^#/;
    chomp $line;
    my ($chr, $arm, $location, $a, $b, $start, $end, $stain) =
        split('\s+',$line);
    next if ($location =~ m/[a-z]$/);
//g;$stain =~ s/
    $start -= 1 if ($start == 1);
    printf "chr%s\t%d\t%d\t%s%s\t%s\n", $chr, $start, $end, $arm, $location,
        $stain;
}

close (FH);
'_EOF_'
    # << happy emacs
    chmod +x ideoToCytoBand.pl
    ./ideoToCytoBand.pl > cytobands.bed

    hgLoadBed -noBin -tab -sqlTable=${HOME}/src/hg/lib/cytoBand.sql \
        hg19 cytoBand cytobands.bed

    hgLoadBed -noBin -tab -sqlTable=${HOME}/src/hg/lib/cytoBandIdeo.sql \
        hg19 cytoBandIdeo cytobands.bed
    #	checking coverage:
    featureBits -noRandom -noHap -countGaps hg19 cytoBand
    #	3095677412 bases of 3095693983 (99.999%) in intersection
    #	that is everything except chrM:
    echo 3095693983-3095677412 | bc -q
    #	16571

##############################################################################
# UCSC to Ensembl chr name mapping (DONE - 2009-05-08 - Hiram)
    #	new names as of Ensembl version 57, see below
    mkdir /hive/data/genomes/hg19/ensembl
    cd /hive/data/genomes/hg19/ensembl
    wget --timestamping \
	'ftp://ftp.ensembl.org/pub/pre/homo_sapiens/GRCh37/dna/*'
    #	do not need the repeat masker sequence (although it would be
    #	interesting to measure to see how it compares)
    rm -f *.dna_rm.*
    #	fortunately we have the same sizes as Ensembl for everything
    #	(except the haplotypes) and the sizes are unique for each sequence
    #	so we can relate the names via their sizes
    mkdir /hive/data/genomes/hg19/bed/ucscToEnsembl
    cd /hive/data/genomes/hg19/bed/ucscToEnsembl
    #	the toplevel file is a duplicate of everything else
    ls /hive/data/genomes/hg19/ensembl/*.fa.gz | grep -v toplevel \
	| while read F
do
    zcat "${F}"
done | faCount stdin > faCount.txt

    cat << '_EOF_' > relateUcscEnsembl.pl
#!/usr/bin/env perl

use strict;
use warnings;

my %ucscChrs;   # key is size, value is UCSC chr name

open (FH,"<../../chrom.sizes") or die "can not read ../../chrom.sizes";
while (my $line = <FH>) {
    chomp $line;
    my ($chr, $size) = split('\s+', $line);
    die "'$line\n'duplicate size in ../chrom.sizes" if (exists($ucscChrs{$size})
);
    $ucscChrs{$size} = $chr;
}
close (FH);

my %ensemblChrs;        # key is size, value is Ensembl chr name

open (FH,"<faCount.txt") or die "can not read faCount.txt";
while (my $line = <FH>) {
    next if ($line =~ m/#/);
    next if ($line =~ m/total/);
    chomp $line;
    my ($chr, $size, $rest) = split('\s+', $line, 3);
    die "'$line\n'duplicate size in faCount.txt" if (exists($ensemblChrs{$size})
);
    $ensemblChrs{$size} = $chr;
}
close (FH);

my %usedUcscChrs;
my %usedEnsemblChrs;
my %ensemblTranslate; # key is Ensembl name, value is UCSC size
foreach my $size (keys %ucscChrs) {
    if (exists($ensemblChrs{$size})) {
        $usedUcscChrs{$size} = $ucscChrs{$size};
        $usedEnsemblChrs{$size} = $ensemblChrs{$size};
        printf "%s\t%s\t%d\n", $ucscChrs{$size}, $ensemblChrs{$size}, $size;
    } else {
        my $ucscName = $ucscChrs{$size};
        my $ensemblName = "unknown";
        if ($ucscName =~ m/^chr6/) {
            $ucscName =~ s/_hap.//;
            $ucscName =~ s/chr6_/chr6_mhc_/;
            $ensemblName = "HS" . uc($ucscName);
        } elsif ($ucscName =~ m/^chr17_/ || $ucscName =~ m/^chr4_/) {
            $ucscName =~ s/_.*/_1/;
            $ensemblName = "HS" . uc($ucscName);
        } elsif ($ucscName =~ m/^chrM/) {
            print "# no translation for chrM\n";
        } else {
            die "unknown UCSC chr name: $ucscName";
        }
        printf "# ucsc $ucscChrs{$size} -> $ensemblName\n";
        $ensemblTranslate{$ensemblName} = $size;
    }
}

foreach my $size (keys %ensemblChrs) {
    if (!exists($usedEnsemblChrs{$size})) {
        my $ensemblName = $ensemblChrs{$size};
        if (! exists($ensemblTranslate{$ensemblName})) {
            die "can not translate Ensembl name $ensemblName";
        } else {
            my $ucscSize = $ensemblTranslate{$ensemblName};
            printf "%s\t%s\t%d\t%d\n", $ucscChrs{$ucscSize}, $ensemblChrs{$size}
, $ucscSize, $size;
        }
    }
}

printf "chrM\tMT\n";
'_EOF_'
    # << happy emacs
    chmod +x relateUcscEnsembl.pl

    ./relateUcscEnsembl.pl  2>&1 | grep -v "^#" \
	| awk '{printf "%s\t%s\n", $1, $2}' | sort > ucscToEnsembl.tab

    cat << '_EOF_' > ucscToEnsembl.sql
# UCSC to Ensembl chr name translation
CREATE TABLE ucscToEnsembl (
    ucsc varchar(255) not null,        # UCSC chromosome name
    ensembl varchar(255) not null,     # Ensembl chromosome name
              #Indices
    PRIMARY KEY(ucsc(21))
);
'_EOF_'

    hgsql hg19 < ucscToEnsembl.sql
    hgsql hg19 \
-e 'LOAD DATA LOCAL INFILE "ucscToEnsembl.tab" INTO TABLE ucscToEnsembl'

    awk '{printf "%s\t%d\n", $2, -$1}' ../../jkStuff/ensGene.haplotype.lift \
	> ensemblLift.tab

    cat << '_EOF_' > ensemblLift.sql
# UCSC offset to Ensembl coordinates
CREATE TABLE ensemblLift (
    chrom varchar(255) not null,      # Ensembl chromosome name
    offset int unsigned not null,     # offset to add to UCSC position 
              #Indices
    PRIMARY KEY(chrom(15))
);
'_EOF_'

    hgsql hg19 < ensemblLift.sql
    hgsql hg19 \
-e 'LOAD DATA LOCAL INFILE "ensemblLift.tab" INTO TABLE ensemblLift'

##############################################################################
# LASTZ MOUSE Mm9 (DONE - 2009-05-13 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzMm9.2009-05-13
    cd /hive/data/genomes/hg19/bed/lastzMm9.2009-05-13

    cat << '_EOF_' > DEF
# human vs mouse
BLASTZ_ABRIDGE_REPEATS=1

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_SMSK=/scratch/data/hg19/linSpecRep/lineageSpecificRepeats
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Mouse Mm9
SEQ2_DIR=/scratch/data/mm9/nib
SEQ2_SMSK=/scratch/data/mm9/notInOthers
SEQ2_LEN=/scratch/data/mm9/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0
    
BASE=/hive/data/genomes/hg19/bed/lastzMm9.2009-05-13
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -syntenicNet \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-chainMinScore=3000 -chainLinearGap=medium > do.log 2>&1 &
    cat fb.hg19.chainMm9Link.txt 
    #	1022734273 bases of 2897316137 (35.299%) in intersection

    #	and the swap
    mkdir /hive/data/genomes/mm9/bed/blastz.hg19.swap
    cd /hive/data/genomes/mm9/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzMm9.2009-05-13/DEF \
	-swap -noLoadChainSplit -syntenicNet \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
    #	real    131m58.763s
    cat fb.mm9.chainHg19Link.txt 
    #	1013880568 bases of 2620346127 (38.693%) in intersection

#########################################################################
# LASTZ Dog CanFam2 (DONE - 2009-05-13 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzCanFam2.2009-05-13
    cd /hive/data/genomes/hg19/bed/lastzCanFam2.2009-05-13

    cat << '_EOF_' > DEF
# human vs dog
BLASTZ_ABRIDGE_REPEATS=1

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_SMSK=/scratch/data/hg19/linSpecRep/lineageSpecificRepeats
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Dog CanFam2
SEQ2_DIR=/scratch/data/canFam2/nib
SEQ2_LEN=/scratch/data/canFam2/chrom.sizes
SEQ2_SMSK=/scratch/scratch/data/canFam2/linSpecRep.notInHuman
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=20000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzCanFam2.2009-05-13
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -syntenicNet \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	-chainMinScore=3000 -chainLinearGap=medium > do.log 2>&1 &
    cat fb.hg19.chainCanFam2Link.txt 
    #	1532073507 bases of 2897316137 (52.879%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/canFam2/bed/blastz.hg19.swap
    cd /hive/data/genomes/canFam2/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzCanFam2.2009-05-13/DEF \
	-noLoadChainSplit -swap \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	-chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
    #	real    200m17.158s
    cat fb.canFam2.chainHg19Link.txt 
    #	1480018167 bases of 2384996543 (62.055%) in intersection
#########################################################################
# LASTZ Chicken GalGal3 (DONE - 2009-05-13 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzGalGal3.2009-05-13
    cd /hive/data/genomes/hg19/bed/lastzGalGal3.2009-05-13

    cat << '_EOF_' > DEF
# human vs chicken
# Specific settings for chicken (per Webb email to Brian Raney)
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=10000
BLASTZ_K=2200
BLASTZ_Q=/scratch/data/blastz/HoxD55.q
BLASTZ_ABRIDGE_REPEATS=1

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_SMSK=/scratch/data/hg19/lineageSpecificRepeats
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Chicken galGal3 - single chunk big enough to run entire chrom
SEQ2_DIR=/scratch/data/galGal3/nib
SEQ2_LEN=/scratch/data/galGal3/chrom.sizes
SEQ2_SMSK=/scratch/data/galGal3/linSpecRep
SEQ2_CHUNK=200000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzGalGal3.2009-05-13
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-syntenicNet \
	-noLoadChainSplit \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	-chainMinScore=5000 -chainLinearGap=loose > do.log 2>&1
    cat fb.hg19.chainGalGal3Link.txt 
    #	104053179 bases of 2897316137 (3.591%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/galGal3/bed/blastz.hg19.swap
    cd /hive/data/genomes/galGal3/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzGalGal3.2009-05-13/DEF \
	-swap \
	-noLoadChainSplit \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	-chainMinScore=5000 -chainLinearGap=loose > swap.log 2>&1
    #	real    16m45.090s
    cat fb.galGal3.chainHg19Link.txt 
    #	91605899 bases of 1042591351 (8.786%) in intersection

#########################################################################
# LASTZ Macaca Mulatta RheMac2 (DONE - 2009-05-13 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzRheMac2.2009-05-13
    cd /hive/data/genomes/hg19/bed/lastzRheMac2.2009-05-13

    cat << '_EOF_' > DEF
# human vs macaca mulatta
BLASTZ=lastz
# maximum M allowed with lastz is only 254
BLASTZ_M=254
BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q
# and place those items here
BLASTZ_O=600
BLASTZ_E=150
# other parameters from panTro2 vs hg18 lastz on advice from Webb
BLASTZ_K=4500
BLASTZ_Y=15000
BLASTZ_T=2

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_IN_CONTIGS=0

# QUERY: Macaca Mulatta RheMac2
SEQ2_DIR=/scratch/data/rheMac2/rheMac2.2bit
SEQ2_LEN=/scratch/data/rheMac2/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LAP=0
SEQ2_IN_CONTIGS=0

BASE=/hive/data/genomes/hg19/bed/lastzRheMac2.2009-05-13
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-syntenicNet \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    760m22.810s
    cat fb.hg19.chainRheMac2Link.txt 
    #	2397361211 bases of 2897316137 (82.744%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/rheMac2/bed/blastz.hg19.swap
    cd /hive/data/genomes/rheMac2/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzRheMac2.2009-05-13/DEF \
	-swap \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> swap.log 2>&1 &
    #	real    83m51.483s
    cat fb.rheMac2.chainHg19Link.txt 
    #	2313806886 bases of 2646704109 (87.422%) in intersection
#########################################################################
# LASTZ Rat Rn4 (DONE - 2009-05-13 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzRn4.2009-05-13
    cd /hive/data/genomes/hg19/bed/lastzRn4.2009-05-13

    cat << '_EOF_' > DEF
# human vs rat
BLASTZ_ABRIDGE_REPEATS=1

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_SMSK=/scratch/data/hg19/lineageSpecificRepeats
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Rat Rn4
SEQ2_DIR=/scratch/data/rn4/nib
SEQ2_SMSK=/scratch/data/rn4/linSpecRep.notInHuman
SEQ2_LEN=/scratch/data/rn4/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0
    
BASE=/hive/data/genomes/hg19/bed/lastzRn4.2009-05-13
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-syntenicNet -noLoadChainSplit \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-chainMinScore=3000 -chainLinearGap=medium > do.log 2>&1 &
    #	real    314m18.227s
    cat fb.hg19.chainRn4Link.txt 
    #	952605822 bases of 2897316137 (32.879%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/rn4/bed/blastz.hg19.swap
    cd /hive/data/genomes/rn4/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzRn4.2009-05-13/DEF \
	-swap -noLoadChainSplit \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
    #	real    188m0.163s
    cat fb.rn4.chainHg19Link.txt 
    #	947862300 bases of 2571531505 (36.860%) in intersection
##############################################################################
# LASTZ Orangutan PonAbe2 (DONE - 2009-05-13 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzPonAbe2.2009-05-13
    cd /hive/data/genomes/hg19/bed/lastzPonAbe2.2009-05-13

    cat << '_EOF_' > DEF
# human vs orangutan
BLASTZ=lastz
# maximum M allowed with lastz is only 254
BLASTZ_M=254
BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q
# and place those items here
BLASTZ_O=600
BLASTZ_E=150
# other parameters from panTro2 vs hg18 lastz on advice from Webb
BLASTZ_K=4500
BLASTZ_Y=15000
BLASTZ_T=2

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_IN_CONTIGS=0

# QUERY: Orangutan PonAbe1
SEQ2_DIR=/scratch/data/ponAbe2/ponAbe2.2bit
SEQ2_LEN=/scratch/data/ponAbe2/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LAP=0
SEQ2_IN_CONTIGS=0

BASE=/hive/data/genomes/hg19/bed/lastzPonAbe2.2009-05-13
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-syntenicNet \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	> do.log 2>&1 &
    cat fb.hg19.chainPonAbe2Link.txt 
    #	2646687531 bases of 2897316137 (91.350%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/ponAbe2/bed/blastz.hg19.swap
    cd /hive/data/genomes/ponAbe2/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzPonAbe2.2009-05-13/DEF \
	-swap \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	> swap.log 2>&1 &
    #	real    124m3.610s
    cat fb.ponAbe2.chainHg19Link.txt 
    #	2772351468 bases of 3093572278 (89.617%) in intersection
##############################################################################
# LASTZ Lamprey PetMar1 (DONE - 2009-05-14 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzPetMar1.2009-05-14
    cd /hive/data/genomes/hg19/bed/lastzPetMar1.2009-05-14

    cat << '_EOF_' > DEF
# Human vs. Lamprey
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_M=50
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=100000000
SEQ1_LAP=10000
SEQ2_LIMIT=5

# QUERY: Lamprey petMar1
SEQ2_DIR=/scratch/data/petMar1/petMar1.2bit
SEQ2_LEN=/scratch/data/petMar1/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LIMIT=300
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzPetMar1.2009-05-14
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=pk \
	> do.log 2>&1 &
    #	real    113m20.116s
    cat fb.hg19.chainPetMar1Link.txt 
    #	31347143 bases of 2897316137 (1.082%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/petMar1/bed/blastz.hg19.swap
    cd /hive/data/genomes/petMar1/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzPetMar1.2009-05-14/DEF \
	-qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=pk \
	-swap > swap.log 2>&1 &
    #	real    59m14.813s
    cat fb.petMar1.chainHg19Link.txt 
    #	26615001 bases of 831696438 (3.200%) in intersection
##############################################################################
# LASTZ Fugu Fr2 (DONE - 2009-05-14 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzFr2.2009-05-14
    cd /hive/data/genomes/hg19/bed/lastzFr2.2009-05-14

    cat << '_EOF_' > DEF
# Human vs. Fugu
# Try "human-fugu" (more distant, less repeat-killed than mammal) params
# +M=50:
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_M=50
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Fugu fr2
#       Align to the scaffolds, results lifed up to chrUn.sdTrf coordinates
SEQ2_DIR=/scratch/data/fr2/fr2.2bit
SEQ2_LEN=/hive/data/genomes/fr2/chrom.sizes
SEQ2_CTGDIR=/hive/data/genomes/fr2/noUn/fr2.scaffolds.2bit
SEQ2_CTGLEN=/hive/data/genomes/fr2/noUn/fr2.scaffolds.sizes
SEQ2_LIFT=/hive/data/genomes/fr2/jkStuff/liftAll.lft
SEQ2_CHUNK=20000000
SEQ2_LIMIT=30
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzFr2.2009-05-14
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=encodek \
	> do.log 2>&1 &
    #	real    5797m9.288s
    #	had a small problem finishing the fundamental batch run, continuing:
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-continue=cat -qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=encodek \
	> cat.log 2>&1 &
    cat fb.hg19.chainFr2Link.txt 
    #	49309456 bases of 2897316137 (1.702%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/fr2/bed/blastz.hg19.swap
    cd /hive/data/genomes/fr2/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzFr2.2009-05-14/DEF \
	-qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=encodek \
	-swap > swap.log 2>&1 &
    #	real    25m8.491s
    cat fb.fr2.chainHg19Link.txt 
    #	42984130 bases of 393312790 (10.929%) in intersection

##############################################################################
# LASTZ Tetraodon TetNig1 (DONE - 2009-05-14 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzTetNig1.2009-05-14
    cd /hive/data/genomes/hg19/bed/lastzTetNig1.2009-05-14

    cat << '_EOF_' > DEF
# human vs tetraodon
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_M=50
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Tetraodon TetNig1 - single chunk big enough to run entire genome
SEQ2_DIR=/scratch/data/tetNig1/tetNig1.2bit
SEQ2_LEN=/hive/data/genomes/tetNig1/chrom.sizes
SEQ2_CHUNK=410000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzTetNig1.2009-05-14
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \
	-verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	> do.log 2>&1 &
    #	real    166m19.745s
    cat fb.hg19.chainTetNig1Link.txt 
    #	58038079 bases of 2897316137 (2.003%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/tetNig1/bed/blastz.hg19.swap
    cd /hive/data/genomes/tetNig1/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzTetNig1.2009-05-14/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	-swap > swap.log 2>&1 &
    #	real    29m20.968s
    cat fb.tetNig1.chainHg19Link.txt 
    #	49453375 bases of 342403326 (14.443%) in intersection

##############################################################################
# LASTZ Stickleback GasAcu1 (DONE - 2009-05-14 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzGasAcu1.2009-05-14
    cd /hive/data/genomes/hg19/bed/lastzGasAcu1.2009-05-14

    cat << '_EOF_' > DEF
# Human vs. Stickleback
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_M=50
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# TARGET: Stickleback gasAcu1
SEQ2_DIR=/scratch/data/gasAcu1/gasAcu1.2bit
SEQ2_LEN=/hive/data/genomes/gasAcu1/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzGasAcu1.2009-05-14
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \
	-verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	> do.log 2>&1 &
    #	real    174m40.659s
    cat fb.hg19.chainGasAcu1Link.txt 
    #	55509003 bases of 2897316137 (1.916%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/gasAcu1/bed/blastz.hg19.swap
    cd /hive/data/genomes/gasAcu1/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzGasAcu1.2009-05-14/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	-swap > swap.log 2>&1 &
    #	real    29m41.433s
    cat fb.gasAcu1.chainHg19Link.txt 
    #	49909819 bases of 446627861 (11.175%) in intersection
##############################################################################
# LASTZ Marmoset CalJac1 (DONE - 2009-05-14,22 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzCalJac1.2009-05-14
    cd /hive/data/genomes/hg19/bed/lastzCalJac1.2009-05-14

    cat << '_EOF_' > DEF
# human vs. marmoset
BLASTZ=lastz
# maximum M allowed with lastz is only 254
BLASTZ_M=254
BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q
# and place those items here
BLASTZ_O=600
BLASTZ_E=150
# other parameters from panTro2 vs hg18 lastz on advice from Webb
BLASTZ_K=4500
BLASTZ_Y=15000
BLASTZ_T=2

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=20000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Marmoset (calJac1)
SEQ2_DIR=/scratch/data/calJac1/calJac1.2bit
SEQ2_LEN=/scratch/data/calJac1/chrom.sizes
SEQ2_LIMIT=200
SEQ2_CHUNK=30000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzCalJac1.2009-05-14
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-syntenicNet \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    214m16.294s
    cat fb.hg19.chainCalJac1Link.txt 
    #	2053025318 bases of 2897316137 (70.860%) in intersection
    time doRecipBest.pl -buildDir=`pwd` hg19 calJac1 > rbest.log 2>&1 &
    #	real    97m17.207s

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/calJac1/bed/blastz.hg19.swap
    cd /hive/data/genomes/calJac1/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzCalJac1.2009-05-14/DEF \
	-swap \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	> swap.log 2>&1 &
    #	real    162m52.189s
    cat fb.calJac1.chainHg19Link.txt 
    #	2105959656 bases of 2929139385 (71.897%) in intersection

#########################################################################
# LASTZ Tarsier TarSyr1 (DONE - 2009-05-14,30 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzTarSyr1.2009-05-14
    cd /hive/data/genomes/hg19/bed/lastzTarSyr1.2009-05-14

    cat << '_EOF_' > DEF
# Human vs. Tarsier

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=200000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Tarsier
SEQ2_DIR=/scratch/data/tarSyr1/tarSyr1.2bit
SEQ2_LEN=/scratch/data/tarSyr1/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LIMIT=50
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzTarSyr1.2009-05-14
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    time nice -n +19 $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \
	-verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    1724m48.032s
    #	need to load the chain table manually:
    #	mySQL error 1114: The table 'chainTarSyr1Link' is full
    cd /hive/data/genomes/hg19/bed/lastzTarSyr1.2009-05-14/axtChain
    wc -l *.tab
    #	 21882142 chain.tab
    #	165017606 link.tab
    #	186899748 total
    awk '{print length($0)}' link.tab | sort | uniq -c | less
      4 23
      9 24
     27 25
    105 26
    767 27
   1401 28
   5020 29
   8472 30
  24390 31
 117666 32
 264774 33
 776095 34
1632393 35
2672187 36
7125988 37
16831901 38
34905113 39
45218159 40
31570706 41
13746548 42
5868689 43
2460114 44
1118556 45
 420826 46
 106674 47
  36770 48
  40719 49
  36955 50
  19389 51
   5571 52
   1557 53
     61 54

    time nice -n +19 hgsql -e "DROP TABLE chainTarSyr1Link;" hg19

    cat << '_EOF_' | hgsql hg19
    CREATE TABLE chainTarSyr1Link (
      bin smallint(5) unsigned NOT NULL default 0,
      tName varchar(255) NOT NULL default '',
      tStart int(10) unsigned NOT NULL default 0,
      tEnd int(10) unsigned NOT NULL default 0,
      qStart int(10) unsigned NOT NULL default 0,
      chainId int(10) unsigned NOT NULL default 0,
      KEY tName (tName(16),bin),
      KEY chainId (chainId)
    ) ENGINE=MyISAM max_rows=166000000 avg_row_length=42 pack_keys=1 CHARSET=latin1;
'_EOF_'
    # << happy emacs

    time nice -n +19 hgsql -e \
      "load data local infile \"link.tab\" into table chainTarSyr1Link;" hg19
    #	real    157m0.230s
    #	the running the rest of loadUp.csh after the hgLoadChain
    #	real    26m8.263s
    cat fb.hg19.chainTarSyr1Link.txt 
    #	1385797066 bases of 2897316137 (47.830%) in intersection
    #	Continuing:
    time nice -n +19 $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \
	-continue=download -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	> download.log 2>&1 &
    #	real    48m6.573s
    #	ran the script on swarm to recover after hive outages
    time doRecipBest.pl -buildDir=`pwd` hg19 tarSyr1 > rbest.log 2>&1 &
    #	real    404m0.201s
    time doRecipBest.pl -continue=download -buildDir=`pwd` \
	hg19 tarSyr1 > rbest.download.log 2>&1 &

#########################################################################
# LASTZ Bushbaby OtoGar1 (DONE - 2009-05-14,22 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzOtoGar1.2009-05-14
    cd /hive/data/genomes/hg19/bed/lastzOtoGar1.2009-05-14

    cat << '_EOF_' > DEF
# Human vs. Bushbaby

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=200000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Bushbaby otoGar1 - single chunk big enough to run largest scaffold
SEQ2_DIR=/scratch/data/otoGar1/otoGar1.rmsk.2bit
SEQ2_LEN=/hive/data/genomes/otoGar1/chrom.sizes
SEQ2_LIMIT=200
SEQ2_CHUNK=30000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzOtoGar1.2009-05-14
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    time nice -n +19 $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \
	-verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    762m56.055s
    cat fb.hg19.chainOtoGar1Link.txt 
    #	1264492372 bases of 2897316137 (43.644%) in intersection
    time doRecipBest.pl -buildDir=`pwd` hg19 otoGar1 > rbest.log 2>&1 &
    #	real    271m39.925s

#########################################################################
# LASTZ Mouse lemur MicMur1 (DONE - 2009-05-14,26 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzMicMur1.2009-05-14
    cd /hive/data/genomes/hg19/bed/lastzMicMur1.2009-05-14

    cat << '_EOF_' > DEF
# Human vs. Mouse lemur

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=200000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Mouse lemur
SEQ2_DIR=/hive/data/genomes/micMur1/bed/repeatMasker/micMur1.rmsk.2bit
SEQ2_LEN=/hive/data/genomes/micMur1/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LIMIT=100
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzMicMur1.2009-05-14
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    time nice -n +19 $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \
	-verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	> do.log 2>&1 &
    #	real    5429m52.082s
    #	there is one unusual long running job having trouble
    #	continuing after finishing the lastz run manually:
    time nice -n +19 $HOME/kent/src/hg/utils/automation/doBlastzChainNet.pl \
	-continue=cat -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	> cat.log 2>&1 &
    #	real    388m25.032s
    cat fb.hg19.chainMicMur1Link.txt 
    #	1347792207 bases of 2897316137 (46.519%) in intersection
    time doRecipBest.pl -buildDir=`pwd` hg19 micMur1 > rbest.log 2>&1
    #	about 4h30m

#########################################################################
# LASTZ Baboon PapHam1 (DONE - 2009-05-20,22 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzPapHam1.2009-05-20
    cd /hive/data/genomes/hg19/bed/lastzPapHam1.2009-05-20

    cat << '_EOF_' > DEF
# human vs baboon
BLASTZ=lastz
# maximum M allowed with lastz is only 254
BLASTZ_M=254
BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q
# and place those items here
BLASTZ_O=600
BLASTZ_E=150
# other parameters from panTro2 vs hg18 lastz on advice from Webb
BLASTZ_K=4500
BLASTZ_Y=15000
BLASTZ_T=2

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=100000000
SEQ1_LAP=10000
SEQ1_IN_CONTIGS=0

# QUERY: Baboon papHam1
SEQ2_DIR=/scratch/data/papHam1/papHam1.2bit
SEQ2_LEN=/scratch/data/papHam1/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LIMIT=300
SEQ2_LAP=0
SEQ2_IN_CONTIGS=0

BASE=/hive/data/genomes/hg19/bed/lastzPapHam1.2009-05-20
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	forgot that the synNet was not needed here, use recip best as below
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-syntenicNet \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    cat fb.hg19.chainPapHam1Link.txt 
    #	2399269031 bases of 2897316137 (82.810%) in intersection

    time doRecipBest.pl -buildDir=`pwd` hg19 papHam1 > rbest.log 2>&1
    #	real    182m0.276s

#########################################################################
# SGP GENES (DONE - 2009-05-22 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/sgpGene
    cd /hive/data/genomes/hg19/bed/sgpGene
    mkdir download
    cd download
for C in `cut -f1 ../../../chrom.sizes`
do
    echo $C
    wget --timestamping \
http://genome.crg.es/genepredictions/H.sapiens/golden_path_200902_x_mm9/SGP/${C}.gtf
    wget --timestamping \
http://genome.crg.es/genepredictions/H.sapiens/golden_path_200902_x_mm9/SGP/${C}.prot
done

    cd ..
    cat download/*.gtf | ldHgGene -gtf -genePredExt hg19 sgpGene stdin

    #	Read 33994 transcripts in 291782 lines in 1 files
    #	33994 groups 85 seqs 1 sources 3 feature types
    #	33994 gene predictions
    nice -n +19 featureBits -enrichment hg19 refGene:CDS sgpGene
# refGene:CDS 1.181%, sgpGene 1.295%, both 1.011%, cover 85.59%, enrich 66.08x

###########################################################################
# GENEID GENE PREDICTIONS (DONE - 2009-05-22 - Hiram)
    ssh hgwdev
    mkdir /hive/data/genomes/hg19/bed/geneid
    cd /hive/data/genomes/hg19/bed/geneid
    mkdir download
    cd download
    for C in `cut -f1 ../../../chrom.sizes`
    do
	echo $C
 wget --timestamping \
http://genome.crg.es/genepredictions/H.sapiens/golden_path_200902/geneid_v1.3/${C}.gtf
    wget --timestamping \
http://genome.crg.es/genepredictions/H.sapiens/golden_path_200902/geneid_v1.3/${C}.prot
    done

    cd ..
    cat download/*.gtf | ldHgGene -gtf -genePredExt hg19 geneid stdin
    #	Read 33428 transcripts in 277332 lines in 1 files
    #	33428 groups 92 seqs 1 sources 3 feature types
    #	33428 gene predictions

##########################################################################
## 4-Way Multiz for UCSC Genes construction (DONE - 2009-05-22 - Hiram)
    ssh hgwdev
    mkdir /hive/data/genomes/hg19/bed/multiz4way
    cd /hive/data/genomes/hg19/bed/multiz4way

    #	extract our 4 organisms from the 44-way on hg18:
    ln -s /hive/data/genomes/hg18/bed/multiz44way/44way.4d.nh ./44way.nh

    /cluster/bin/phast/tree_doctor \
	--prune-all-but hg18,mm9,canFam2,rheMac2 44way.nh \
	| sed -e "s/hg18/hg19/" > 4way.nh

    #	this looks like:
    cat 4way.nh
(((hg19:0.032973,rheMac2:0.036199):0.109706,mm9:0.352605):0.020666,canFam2:0.193569);


    #	Use this specification in the phyloGif tool:
    #	http://genome.ucsc.edu/cgi-bin/phyloGif
    #	to obtain a gif image for htdocs/images/phylo/hg19_4way.gif

    /cluster/bin/phast/all_dists 4way.nh > 4way.distances.txt
    #	Use this output to create the table below
    grep -y hg19 4way.distances.txt | sort -k3,3n
#
#	If you can fill in all the numbers in this table, you are ready for
#	the multiple alignment procedure
#
#                         featureBits chainLink measures
#                                        chainHg19Link   chain    linearGap
#    distance                      on hg19    on other   minScore
#  1  0.069172 - rhesus rheMac2 (% 82.744) (% xx.xxx)       5000     medium
#  2  0.356914 - dog canFam2    (% 52.879) (% xx.xxx)       3000     medium
#  3  0.495284 - mouse mm9      (% 35.299) (% 38.693)       3000     medium

    #	using the syntenic nets
    cd /cluster/data/hg19/bed/multiz4way
    mkdir mafLinks
    cd mafLinks
    mkdir rheMac2 canFam2 mm9

    cd mm9
    ln -s ../../../lastz.mm9/mafSynNet/*.maf.gz .
    cd ../canFam2
    ln -s ../../../lastz.canFam2/mafSynNet/*.maf.gz .
    cd ../rheMac2
    ln -s ../../../lastz.rheMac2/mafSynNet/*.maf.gz .

    #	determine what is the newest version of multiz and use that
    cd /hive/data/genomes/hg19/bed/multiz4way
    mkdir penn
    cp -p /cluster/bin/penn/multiz.v11.2007-03-19/multiz-tba/multiz penn
    cp -p /cluster/bin/penn/multiz.v11.2007-03-19/multiz-tba/maf_project penn
    cp -p /cluster/bin/penn/multiz.v11.2007-03-19/multiz-tba/autoMZ penn

    # the autoMultiz cluster run
    ssh swarm
    cd /hive/data/genomes/hg19/bed/multiz4way

    # create species list and stripped down tree for autoMZ
    sed 's/[a-z][a-z]*_//g; s/:[0-9\.][0-9\.]*//g; s/;//; /^ *$/d' \
	4way.nh > tmp.nh
    echo `cat tmp.nh` | sed 's/ //g; s/,/ /g' > tree.nh
    sed 's/[()]//g; s/,/ /g' tree.nh > species.lst

    mkdir run maf
    cd run

    #	NOTE: you need to set the db and multiz dirname properly in this script
    cat > autoMultiz << '_EOF_'
#!/bin/csh -ef
set db = hg19
set c = $1
set maf = $2
set binDir = /hive/data/genomes/hg19/bed/multiz4way/penn
set tmp = /scratch/tmp/$db/multiz.$c
set pairs = /hive/data/genomes/hg19/bed/multiz4way/mafLinks
rm -fr $tmp
mkdir -p $tmp
cp ../{tree.nh,species.lst} $tmp
pushd $tmp
foreach s (`cat species.lst`)
    set in = $pairs/$s/$c.maf
    set out = $db.$s.sing.maf
    if ($s == $db) then
	continue
    endif
    if (-e $in.gz) then
	zcat $in.gz > $out
    else if (-e $in) then
	cp $in $out
    else
	echo "##maf version=1 scoring=autoMZ" > $out
    endif
end
set path = ($binDir $path); rehash
$binDir/autoMZ + T=$tmp E=$db "`cat tree.nh`" $db.*.sing.maf $c.maf
popd
cp $tmp/$c.maf $maf
rm -fr $tmp
'_EOF_'
    # << happy emacs
    chmod +x autoMultiz

cat  << '_EOF_' > template
#LOOP
./autoMultiz $(root1) {check out line+ /hive/data/genomes/hg19/bed/multiz4way/maf/$(root1).maf}
#ENDLOOP
'_EOF_'
    # << happy emacs

    cut -f1 /cluster/data/hg19/chrom.sizes > chrom.lst
    gensub2 chrom.lst single template jobList
    para create jobList
    # 93 jobs
    para try ... check ... push ... etc ...
# Completed: 93 of 93 jobs
# CPU time in finished jobs:      24282s     404.70m     6.75h    0.28d  0.001 y
# IO & Wait Time:                  2362s      39.36m     0.66h    0.03d  0.000 y
# Average job time:                 286s       4.77m     0.08h    0.00d
# Longest finished job:            2235s      37.25m     0.62h    0.03d
# Submission to last job:          2241s      37.35m     0.62h    0.03d

    #	combine results into a single file for loading and gbdb reference
    cd /hive/data/genomes/hg19/bed/multiz4way
    time nice -n +19 catDir maf > multiz4way.maf
    #	real    3m27.561s

    #	makes a 8.5 Gb file:
    #	-rw-rw-r-- 1 9026080732 May 22 11:11 multiz4way.maf

    # Load into database
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/multiz4way
    mkdir /gbdb/hg19/multiz4way
    ln -s /hive/data/genomes/hg19/bed/multiz4way/multiz4way.maf \
	/gbdb/hg19/multiz4way
    #	the hgLoadMaf generates huge tmp files, locate them in /scratch/tmp/
    cd /scratch/tmp
    time nice -n +19 hgLoadMaf hg19 multiz4way
    #	real    5m31.883s
    #	Loaded 5788627 mafs in 1 files from /gbdb/hg19/multiz4way

    cd /hive/data/genomes/hg19/bed/multiz4way
    time nice -n +19 hgLoadMafSummary -minSize=10000 -mergeGap=500 \
	-maxSize=50000 hg19 multiz4waySummary multiz4way.maf
    #	Created 1238721 summary blocks from 11959676 components
    #	and 5788627 mafs from multiz4way.maf
    #	real    6m33.936s

#########################################################################
# LASTZ Medaka OryLat2 (DONE - 2009-05-22 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzOryLat2.2009-05-22
    cd /hive/data/genomes/hg19/bed/lastzOryLat2.2009-05-22

    cat << '_EOF_' > DEF
# Human vs. Medaka
# typical parameters for a genome that is distant from human
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_M=50
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Medaka oryLat2 (40M chunks covers the largest chroms in one gulp)
SEQ2_DIR=/scratch/data/oryLat2/oryLat2.2bit
SEQ2_LEN=/hive/data/genomes/oryLat2/chrom.sizes
SEQ2_CHUNK=40000000
SEQ2_LIMIT=200
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzOryLat2.2009-05-22
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	> do.log 2>&1 &
    #	real    124m5.298s
    cat fb.hg19.chainOryLat2Link.txt 
    #	53571737 bases of 2897316137 (1.849%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/oryLat2/bed/blastz.hg19.swap
    cd /hive/data/genomes/oryLat2/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzOryLat2.2009-05-22/DEF \
	-qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	-swap > swap.log 2>&1 &
    #	real    28m35.174s
    cat fb.oryLat2.chainHg19Link.txt 
    #	46961818 bases of 700386597 (6.705%) in intersection
##############################################################################
# LASTZ Opossum MonDom5 (DONE - 2009-05-23,29 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzMonDom5.2009-05-23
    cd /hive/data/genomes/hg19/bed/lastzMonDom5.2009-05-23

    cat << '_EOF_' > DEF
# human vs. opossum
# settings for more distant organism alignments
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_M=50
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Opossum monDom5
SEQ2_DIR=/scratch/data/monDom5/monDom5.2bit
SEQ2_LEN=/hive/data/genomes/monDom5/chrom.sizes
SEQ2_CHUNK=30000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzMonDom5.2009-05-23
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	One job took a long time to complete, had to run it manually on
    #	swarm:
# /cluster/bin/scripts/blastz-run-ucsc -outFormat psl \
#	/scratch/data/hg19/hg19.2bit:chr19:50000000-59128983 \
#	/scratch/data/monDom5/monDom5.2bit:chr4:390000000-420000000 \
#	../DEF \
#	../psl/hg19.2bit:chr19:50000000-59128983/hg19.2bit:chr19:50000000-59128983_monDom5.2bit:chr4:390000000-420000000.psl
    #	took about 48 hours, continuing:
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-continue=cat > cat.log 2>&1 &
    #	real    1508m18.471s ==	about 25h08m
    cat fb.hg19.chainMonDom5Link.txt 
    #	415997117 bases of 2897316137 (14.358%) in intersection
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-continue=syntenicNet -syntenicNet > syntenicNet.log 2>&1 &
    #	real    20m29.049s

    mkdir /hive/data/genomes/monDom5/bed/blastz.hg19.swap
    cd /hive/data/genomes/monDom5/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzMonDom5.2009-05-23/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-swap -syntenicNet > swap.log 2>&1 &
    #	real    297m13.041s
    cat fb.monDom5.chainHg19Link.txt 
    #	406727849 bases of 3501660299 (11.615%) in intersection

##############################################################################
# LASTZ Armadillo DasNov2 (DONE - 2009-05-23,28 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzDasNov2.2009-05-23
    cd /hive/data/genomes/hg19/bed/lastzDasNov2.2009-05-23

    cat << '_EOF_' > DEF
# Human vs. Armadillo
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Armadillo
SEQ2_DIR=/scratch/data/dasNov2/dasNov2.2bit
SEQ2_LEN=/scratch/data/dasNov2/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LIMIT=100
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzDasNov2.2009-05-23
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	> do.log 2>&1 &
    #	finished the lastz run manually after hive maintenance outages
    #	then, continuing:
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	-continue=cat > cat.log 2>&1 &
    #	real    458m11.304s
    cat fb.hg19.chainDasNov2Link.txt 
    #	971847303 bases of 2897316137 (33.543%) in intersection
    time nice -n +19 doRecipBest.pl -buildDir=`pwd` hg19 dasNov2 \
	> rbest.log 2>&1
    #	time about 6h30m

##############################################################################
# LASTZ Rock Hyrax ProCap1 (DONE - 2009-05-23,26 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzProCap1.2009-05-23
    cd /hive/data/genomes/hg19/bed/lastzProCap1.2009-05-23

    cat << '_EOF_' > DEF
# Human vs. Rock Hyrax
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Rock Hyrax
SEQ2_DIR=/scratch/data/proCap1/proCap1.2bit
SEQ2_LEN=/scratch/data/proCap1/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LIMIT=100
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzProCap1.2009-05-23
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	> do.log 2>&1 &
# Completed: 997438 of 997438 jobs
# CPU time in finished jobs:   32830587s  547176.45m  9119.61h  379.98d  1.041 y
# IO & Wait Time:               9549484s  159158.07m  2652.63h  110.53d  0.303 y
# Average job time:                  42s       0.71m     0.01h    0.00d
# Longest finished job:            1953s      32.55m     0.54h    0.02d
# Submission to last job:         67216s    1120.27m    18.67h    0.78d
    #	finished lastz run manually, then continuing:
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-continue=cat > cat.log 2>&1 &
    #	real    369m1.678s
    cat fb.hg19.chainProCap1Link.txt 
    #	894221652 bases of 2897316137 (30.864%) in intersection
    time nice -n +19 doRecipBest.pl -buildDir=`pwd` hg19 proCap1 \
	> rbest.log 2>&1
    #	real    251m59.549s

##############################################################################
# LASTZ Zebra Finch TaeGut1 (DONE - 2009-05-26 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzTaeGut1.2009-05-26
    cd /hive/data/genomes/hg19/bed/lastzTaeGut1.2009-05-26

    cat << '_EOF_' > DEF
# human vs Zebra Finch
# distant from Human settings
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=10000
BLASTZ_K=2200
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Zebra Finch taeGut1 - single chunk big enough to run entire chrom
SEQ2_DIR=/scratch/data/taeGut1/taeGut1.2bit
SEQ2_LEN=/scratch/data/taeGut1/chrom.sizes
SEQ2_CTGDIR=/hive/data/genomes/taeGut1/taeGut1.blastz.2bit
SEQ2_CTGLEN=/hive/data/genomes/taeGut1/taeGut1.blastz.sizes
SEQ2_LIFT=/hive/data/genomes/taeGut1/jkStuff/liftAll.lft
SEQ2_CHUNK=20000000
SEQ2_LAP=0
SEQ2_LIMIT=100

BASE=/hive/data/genomes/hg19/bed/lastzTaeGut1.2009-05-26
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-qRepeats=windowmaskerSdust > do.log 2>&1 &
    cat fb.hg19.chainTaeGut1Link.txt 
    #	real    192m48.479s
    #	101295490 bases of 2897316137 (3.496%) in intersection
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-syntenicNet -noLoadChainSplit -chainMinScore=5000 \
	-chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-continue=syntenicNet -qRepeats=windowmaskerSdust > synNet.log 2>&1 &
    #	real    4m10.261s

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/taeGut1/bed/blastz.hg19.swap
    cd /hive/data/genomes/taeGut1/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzTaeGut1.2009-05-26/DEF \
	-swap -noLoadChainSplit -chainMinScore=5000 \
	-chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-qRepeats=windowmaskerSdust > swap.log 2>&1 &
    #	real    real    16m45.080s
    cat fb.taeGut1.chainHg19Link.txt 
    #	95320369 bases of 1222864691 (7.795%) in intersection

##############################################################################
# LASTZ Lizard AnoCar1 (DONE - 2009-05-30,31 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzAnoCar1.2009-05-30
    cd /hive/data/genomes/hg19/bed/lastzAnoCar1.2009-05-30

    cat << '_EOF_' > DEF
# human vs lizard
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Lizard anoCar1
SEQ2_DIR=/scratch/data/anoCar1/anoCar1.2bit
SEQ2_LEN=/scratch/data/anoCar1/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LAP=0
SEQ2_LIMIT=50

BASE=/hive/data/genomes/hg19/bed/lastzAnoCar1.2009-05-30
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-qRepeats=windowmaskerSdust > do.log 2>&1 &
    #	real    168m32.016s
    cat fb.hg19.chainAnoCar1Link.txt 
    #	104045950 bases of 2897316137 (3.591%) in intersection
    time doRecipBest.pl -buildDir=`pwd` hg19 anoCar1 > rbest.log 2>&1
    #	real    45m58.001s

    #	running syntenic Net 2009-08-27 - Hiram
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-continue=syntenicNet -syntenicNet \
	-qRepeats=windowmaskerSdust > syntenicNet.log 2>&1 &
    #	real    6m13.304s

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/anoCar1/bed/blastz.hg19.swap
    cd /hive/data/genomes/anoCar1/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzAnoCar1.2009-05-30/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-swap -qRepeats=windowmaskerSdust > swap.log 2>&1 &
    #	real    34m55.857s
    cat fb.anoCar1.chainHg19Link.txt 
    #	89608316 bases of 1741478929 (5.146%) in intersection
##############################################################################
# LASTZ X. tropicalis XenTro2 (DONE - 2009-05-26 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzXenTro2.2009-05-26
    cd /hive/data/genomes/hg19/bed/lastzXenTro2.2009-05-26

    cat << '_EOF_' > DEF
# human vs X. tropicalis
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Lizard anoCar1
SEQ2_DIR=/scratch/data/xenTro2/xenTro2.2bit
SEQ2_LEN=/scratch/data/xenTro2/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LAP=0
SEQ2_LIMIT=100

BASE=/hive/data/genomes/hg19/bed/lastzXenTro2.2009-05-26
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    1129m11.568s
    #	finished the lastz run manually after hive difficulties, continuing:
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-continue=cat > cat.log 2>&1 &
    #	time about 1h30m
    cat fb.hg19.chainXenTro2Link.txt 
    #	92015242 bases of 2897316137 (3.176%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/xenTro2/bed/blastz.hg19.swap
    cd /hive/data/genomes/xenTro2/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzXenTro2.2009-05-26/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-swap > swap.log 2>&1 &
    #	real    130m53.860s
    cat fb.xenTro2.chainHg19Link.txt 
    #	92070065 bases of 1359412157 (6.773%) in intersection

##############################################################################
# LASTZ Zebrafish DanRer5 (DONE - 2009-05-26 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzDanRer5.2009-05-26
    cd /hive/data/genomes/hg19/bed/lastzDanRer5.2009-05-26

    cat << '_EOF_' > DEF
# human vs X. zebrafish
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Zebrafish danRer5
SEQ2_DIR=/scratch/data/danRer5/danRer5.2bit
SEQ2_LEN=/scratch/data/danRer5/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LAP=0
SEQ2_LIMIT=40

BASE=/hive/data/genomes/hg19/bed/lastzDanRer5.2009-05-26
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    311m39.817s
    cat fb.hg19.chainDanRer5Link.txt 
    #	74229561 bases of 2897316137 (2.562%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/danRer5/bed/blastz.hg19.swap
    cd /hive/data/genomes/danRer5/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzDanRer5.2009-05-26/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-swap > swap.log 2>&1 &
    #	real    26m54.605s
    cat fb.danRer5.chainHg19Link.txt 
    #	73852780 bases of 1435609608 (5.144%) in intersection

##############################################################################
# LASTZ Platypus OrnAna1 (DONE - 2009-05-26 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzOrnAna1.2009-05-26
    cd /hive/data/genomes/hg19/bed/lastzOrnAna1.2009-05-26

    cat << '_EOF_' > DEF
# human vs platypus
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Platypus ornAna1
SEQ2_DIR=/scratch/data/ornAna1/ornAna1.2bit
SEQ2_LEN=/scratch/data/ornAna1/chrom.sizes
SEQ2_CHUNK=40000000
SEQ2_LIMIT=400
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzOrnAna1.2009-05-26
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    572m18.808s
    cat fb.hg19.chainOrnAna1Link.txt 
    #	220977689 bases of 2897316137 (7.627%) in intersection
    time doRecipBest.pl -buildDir=`pwd` hg19 ornAna1 > rbest.log 2>&1
    #	time about 1h32m

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/ornAna1/bed/blastz.hg19.swap
    cd /hive/data/genomes/ornAna1/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzOrnAna1.2009-05-26/DEF \
	-swap -noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> swap.log 2>&1 &
    #	real    146m52.638s
    cat fb.ornAna1.chainHg19Link.txt 
    #	207415519 bases of 1842236818 (11.259%) in intersection

##############################################################################
# LASTZ Elephant LoxAfr2 (DONE - 2009-05-27,29 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzLoxAfr2.2009-05-27
    cd /hive/data/genomes/hg19/bed/lastzLoxAfr2.2009-05-27

    cat << '_EOF_' > DEF
# Human vs. Elephant
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Elephant
SEQ2_DIR=/scratch/data/loxAfr2/loxAfr2.2bit
SEQ2_LEN=/scratch/data/loxAfr2/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LIMIT=300
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzLoxAfr2.2009-05-27
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    # time about 3h23m
    cat fb.hg19.chainLoxAfr2Link.txt 
    #	1018502258 bases of 2897316137 (35.153%) in intersection

    time doRecipBest.pl -buildDir=`pwd` hg19 loxAfr2 > rbest.log 2>&1
    #	real    322m37.502s

##############################################################################
# LASTZ Tenrec EchTel1 (DONE - 2009-05-27 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzEchTel1.2009-05-27
    cd /hive/data/genomes/hg19/bed/lastzEchTel1.2009-05-27

    cat << '_EOF_' > DEF
# Human vs. Tenrec
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Tenrec
SEQ2_DIR=/scratch/data/echTel1/echTel1.2bit
SEQ2_LEN=/scratch/data/echTel1/chrom.sizes
SEQ2_CHUNK=30000000
SEQ2_LIMIT=400
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzEchTel1.2009-05-27
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    1153m34.595s
    cat fb.hg19.chainEchTel1Link.txt 
    #	669856841 bases of 2897316137 (23.120%) in intersection
    time doRecipBest.pl -buildDir=`pwd` hg19 echTel1 > rbest.log 2>&1
    # time about 7h13m

##############################################################################
# LASTZ Tree Shrew TupBel1 (DONE - 2009-05-27,06-02 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzTupBel1.2009-05-27
    cd /hive/data/genomes/hg19/bed/lastzTupBel1.2009-05-27

    cat << '_EOF_' > DEF
# Human vs. Tree Shrew
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Tree Shrew
SEQ2_DIR=/scratch/data/tupBel1/tupBel1.2bit
SEQ2_LEN=/scratch/data/tupBel1/chrom.sizes
SEQ2_CHUNK=30000000
SEQ2_LIMIT=400
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzTupBel1.2009-05-27
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=pk \
	> do.log 2>&1 &
    #	real 811m54.095s
    #	having trouble with pk, finished manually
    #	XXX there is one job that is taking forever ...
    #	finished it in pieces on swarm in a few minutes, like this:
    mkdir /hive/data/genomes/hg19/bed/lastzTupBel1.2009-05-27/run.blastz/lastJob
    cd /hive/data/genomes/hg19/bed/lastzTupBel1.2009-05-27/run.blastz/lastJob
#!/bin/sh

S=100000000
E=101010000
export S E
for I in 0 1 2 3 4 5 6 7 8 9
do
echo $S $E
/usr/bin/time -p /cluster/bin/scripts/blastz-run-ucsc -outFormat psl \
/scratch/data/hg19/nib/chr1.nib:chr1:${S}-${E} ../qParts/part019.lst \
../../DEF psl/chr1.nib:chr1:${S}-${E}_part019.lst.psl
nextS=`echo $S | awk '{printf "%d", $1 + 1000000}'`
nextE=`echo $E | awk '{printf "%d", $1 + 1000000}'`
S=$nextS
E=$nextE
done

    grep -h "^#" psl/chr* | sort -u > result.psl
    grep -h -v "^#" psl/chr* | sort -k14,14 -k16,16n >> result.psl
    cp -p result.psl \
../../psl/chr1.nib:chr1:100000000-110010000/chr1.nib:chr1:100000000-110010000_part019.lst.psl

    #	then, continuing:
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=pk \
	-continue=cat > cat.log 2>&1 &
    #	real    212m22.707s
    time doRecipBest.pl -buildDir=`pwd` hg19 tupBel1 > rbest.log 2>&1
    #	time about 4h22m

##############################################################################
# LASTZ Shrew SorAra1 (DONE - 2009-05-28,30 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzSorAra1.2009-05-28
    cd /hive/data/genomes/hg19/bed/lastzSorAra1.2009-05-28

    cat << '_EOF_' > DEF
# Human vs. Shrew
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Shrew
SEQ2_DIR=/scratch/data/sorAra1/sorAra1.2bit
SEQ2_LEN=/scratch/data/sorAra1/chrom.sizes
SEQ2_CHUNK=30000000
SEQ2_LIMIT=400
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzSorAra1.2009-05-28
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	time about 23h26m
    cat fb.hg19.chainSorAra1Link.txt 
    #	572519288 bases of 2897316137 (19.760%) in intersection

    time doRecipBest.pl -buildDir=`pwd` hg19 sorAra1 > rbest.log 2>&1
    #	real    251m20.055s

##############################################################################
# LASTZ Rabbit OryCun1 (DONE - 2009-05-28,30 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzOryCun1.2009-05-28
    cd /hive/data/genomes/hg19/bed/lastzOryCun1.2009-05-28

    cat << '_EOF_' > DEF
# Human vs. Rabbit
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Rabbit
SEQ2_DIR=/scratch/data/oryCun1/oryCun1.2bit
SEQ2_LEN=/scratch/data/oryCun1/chrom.sizes
SEQ2_CHUNK=30000000
SEQ2_LIMIT=400
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzOryCun1.2009-05-28
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	time about 23h09m
    cat fb.hg19.chainOryCun1Link.txt 
    #	975693323 bases of 2897316137 (33.676%) in intersection

    time doRecipBest.pl -buildDir=`pwd` hg19 oryCun1 > rbest.log 2>&1
    #	real    318m1.142s

##############################################################################
# LASTZ Hedgehog EriEur1 (DONE - 2009-05-28,30 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzEriEur1.2009-05-28
    cd /hive/data/genomes/hg19/bed/lastzEriEur1.2009-05-28

    cat << '_EOF_' > DEF
# Human vs. Hedgehog
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Hedgehog
SEQ2_DIR=/scratch/data/eriEur1/eriEur1.2bit
SEQ2_LEN=/scratch/data/eriEur1/chrom.sizes
SEQ2_CHUNK=40000000
SEQ2_LIMIT=500
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzEriEur1.2009-05-28
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	> do.log 2>&1 &
    #	real    2043m33.198s
    cat fb.hg19.chainEriEur1Link.txt 
    #	560965051 bases of 2897316137 (19.362%) in intersection
    time doRecipBest.pl -buildDir=`pwd` hg19 eriEur1 > rbest.log 2>&1
    #	real    350m17.737s

##############################################################################
# LASTZ Pika OchPri2 (DONE - 2009-05-29,30 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzOchPri2.2009-05-29
    cd /hive/data/genomes/hg19/bed/lastzOchPri2.2009-05-29

    cat << '_EOF_' > DEF
# Human vs. Pika
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Pika
SEQ2_DIR=/scratch/data/ochPri2/ochPri2.2bit
SEQ2_LEN=/scratch/data/ochPri2/chrom.sizes
SEQ2_CHUNK=40000000
SEQ2_LIMIT=400
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzOchPri2.2009-05-29
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    393m42.569s
    cat fb.hg19.chainOchPri2Link.txt 
    #	804516397 bases of 2897316137 (27.768%) in intersection
    time doRecipBest.pl -buildDir=`pwd` hg19 ochPri2 > rbest.log 2>&1
    #	real    224m47.979s

##############################################################################
# LASTZ Kangaroo Rat DipOrd1 (DONE - 2009-05-29,30 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzDipOrd1.2009-05-29
    cd /hive/data/genomes/hg19/bed/lastzDipOrd1.2009-05-29

    cat << '_EOF_' > DEF
# Human vs. Kangaroo Rat
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Kangaroo Rat
SEQ2_DIR=/scratch/data/dipOrd1/dipOrd1.2bit
SEQ2_LEN=/scratch/data/dipOrd1/chrom.sizes
SEQ2_CHUNK=30000000
SEQ2_LIMIT=300
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzDipOrd1.2009-05-29
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    688m47.595s
    time doRecipBest.pl -buildDir=`pwd` hg19 dipOrd1 > rbest.log 2>&1
    #	real    140m42.014s

##############################################################################
# LIFTOVER TO Hg18 (DONE - 2009-06-04 - Hiram )
    mkdir /hive/data/genomes/hg19/bed/blat.hg18.2009-06-04
    cd /hive/data/genomes/hg19/bed/blat.hg18.2009-06-04
    # -debug run to create run dir, preview scripts...
    #	verifies files can be found
    doSameSpeciesLiftOver.pl -debug hg19 hg18
    # Real run:
    time nice -n +19 doSameSpeciesLiftOver.pl -verbose=2 \
	-bigClusterHub=swarm -dbHost=hgwdev -workhorse=hgwdev \
	 hg19 hg18 > do.log 2>&1
    #	real    115m26.071s

#############################################################################
# BLASTZ/CHAIN/NET/ETC 11 GENOMES TO HG19 (DONE, Andy 2009-06-06)
ssh hgwdev
cd /hive/data/genomes/hg19/bed
mkdir lastz{SpeTri1,FelCat3,CavPor3,BosTau4,PteVam1,EquCab2,VicPac1,MyoLuc1,TurTru1,ChoHof1}.2009-06-04
ln -s lastzSpeTri1.2009-06-04 lastz.speTri1
ln -s lastzFelCat3.2009-06-04 lastz.felCat3
ln -s lastzCavPor3.2009-06-04 lastz.cavPor3
ln -s lastzBosTau4.2009-06-04 lastz.bosTau4
ln -s lastzPteVam1.2009-06-04 lastz.pteVam1
ln -s lastzEquCab2.2009-06-04 lastz.equCab2
ln -s lastzVicPac1.2009-06-04 lastz.vicPac1
ln -s lastzMyoLuc1.2009-06-04 lastz.myoLuc1
ln -s lastzTurTru1.2009-06-04 lastz.turTru1
ln -s lastzChoHof1.2009-06-04 lastz.choHof1
cat > lastz.speTri1/DEF << 'EOF'
# human vs squirrel

# TARGET: human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: squirrel speTri1 
SEQ2_DIR=/hive/data/genomes/speTri1/speTri1.2bit
SEQ2_LEN=/hive/data/genomes/speTri1/chrom.sizes
SEQ2_LIMIT=100
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastz.speTri1
TMPDIR=/scratch/tmp
EOF

sed 's/speTri1/felCat3/g; s/squirrel/cat/;' lastz.speTr1/DEF | \
   sed 's/SEQ1_CHUNK=1/SEQ1_CHUNK=2/; s/SEQ2_LIMIT=1/SEQ2_LIMIT=3/' \
   > lastz.felCat3/DEF
sed 's/speTri1/cavPor3/g; s/squirrel/guinea pig/;' lastz.speTr1/DEF | \
   sed 's/SEQ1_CHUNK=1/SEQ1_CHUNK=2/' | \
   sed 's/hive\/data\/genomes\/cavPor3/scratch\/data\/cavPor3/' \
   > lastz.cavPor3/DEF
sed 's/speTri1/bosTau4/g; s/squirrel/cow/;' lastz.speTr1/DEF | \
   sed 's/SEQ1_CHUNK=1/SEQ1_CHUNK=2/; s/SEQ2_LIMIT=1/SEQ2_LIMIT=3/' \
   > lastz.bosTau4/DEF
sed 's/speTri1/pteVam1/g; s/squirrel/megabat/;' lastz.speTr1/DEF | \
   sed 's/SEQ1_CHUNK=1/SEQ1_CHUNK=2/; s/SEQ2_LIMIT=1/SEQ2_LIMIT=2/' \
   > lastz.pteVam1/DEF
sed 's/cavPor3/equCab2/g; s/guinea pig/horse/' lastz.cavPor3/DEF | \ 
   sed 's/SEQ2_LIMIT=1/SEQ2_LIMIT=3/' > lastz.equCab2/DEF
sed 's/equCab2/vicPac1/g; s/horse/alpaca/' lastz.equCab2/DEF > lastz.vicPac1/DEF
sed 's/pteVam1/myoLuc1/g; s/megabat/microbat/' lastz.pteVam1/DEF | \
   sed 's/SEQ2_LIMIT=3/SEQ2_LIMIT=2/' > lastz.myoLuc1/DEF
sed 's/equCab2/turTru1/g; s/horse/dolphin/' lastz.equCab2/DEF | \
   sed 's/SEQ2_LIMIT=3/SEQ2_LIMIT=2/' > lastz.turTru1/DEF
sed 's/equCab2/choHof11/g; s/horse/sloth/' lastz.equCab2/DEF > lastz.choHof1/DEF

cd andy/
for db in speTri1 felCat3 cavPor3 bosTau4 pteVam1 equCab2 vicPac1 myoLuc1 turTru1 choHof1; do
    ln -s ../lastz.${db}/DEF ${db}.DEF
done

screen -S speTri1
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit \
  -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
  -chainMinScore=3000 -chainLinearGap=medium speTri1.DEF >& speTri1.do.log
# [detach screen]
#real    2059m30.699s

screen -S felCat3
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit \
  -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
  -chainMinScore=3000 -chainLinearGap=medium felCat3.DEF >& felCat3.do.log
# [detach screen]
#real    1574m47.522s

screen -S bosTau4
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit \
  -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
  -chainMinScore=3000 -chainLinearGap=medium bosTau4.DEF >& bosTau4.do.log
# [detach screen]
#real    1474m54.655s

screen -S pteVam1
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit \
  -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm
  -chainMinScore=3000 -chainLinearGap=medium pteVam1.DEF >& pteVam1.do.log
# [detach screen]
#real    1168m33.923s

screen -S equCab2
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit \
  -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
  -chainMinScore=3000 -chainLinearGap=medium -syntenicNet equCab2.DEF >& equCab2.do.log
# [detach screen]
#real    1662m56.158s
# (included syntenic net)

screen -S vicPac1
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit \
  -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
  -chainMinScore=3000 -chainLinearGap=medium vicPac1.DEF >& vicPac1.do.log
# [detach screen]
#real    1495m48.173s

screen -S turTru1
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit \
  -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
  -chainMinScore=3000 -chainLinearGap=medium turTru1.DEF >& turTru1.do.log
# [detach screen]
#real    1079m17.234s

screen -S choHof1
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit \
  -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
  -chainMinScore=3000 -chainLinearGap=medium choHof1.DEF >& choHof1.do.log
# [detach screen]
#real    1310m49.287s (script and cluster run stopped after halfway...
# pk was too slow... remaining jobs started on swarm)
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit \
  -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
  -chainMinScore=3000 -chainLinearGap=medium -continue=cat \
  choHof1.DEF >& choHof1.doAfterBlastz.log
#real    257m32.701s

screen -S cavPor3
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit \
  -workhorse=hgwdev -chainMinScore=3000 -chainLinearGap=medium \
  -smallClusterHub=memk -bigClusterHub=pk cavPor3.DEF >& cavPor3.do.log
# [detach screen]
#real    1370m5.258s
# TROUBLE!  got to the 'load' step and failed.  This one needs a special
# chain table and chainLink table to get loaded.
cd ../lastz.cavPor3/axtChain/
# figure out number of rows and average length
wc -l *.tab
#   27186468 chain.tab
#  240602108 link.tab
randomLines link.tab 10000000 stdout | awk '{print length($0)}' | sort | uniq -c
randomLines chain.tab 1000000 stdout | awk '{print length($0)}' | sort | uniq -c
# about 43 average length for the chainLink and 100 for the chain
sed "s/hgLoadChain.*/hgsqldump hg19 chainSpeTri1Link --no-data --skip-comments | sed \'s\/SpeTri1\/CavPor3\/; s\/TYPE=MyISAM\/ENGINE=MyISAM max_rows=241000000 avg_row_length=43 pack_keys=1 CHARSET=latin1\/\' | hgsql hg19 \n\
hgsqldump hg19 chainSpeTri1 --no-data --skip-comments | sed \'s\/SpeTri1\/CavPor3\/; s\/TYPE=MyISAM\/ENGINE=MyISAM max_rows=27200000 avg_row_length=100 pack_keys=1 CHARSET=latin1\/\' | hgsql hg19 \n\
hgsql hg19 -e \"load data local infile \'chain.tab\' into table chainCavPor3\"\n\
hgsql hg19 -e \"load data local infile \'link.tab\' into table chainCavPor3Link\"\n\
hgsql hg19 -e \"INSERT into history (ix, startId, endId, who, what, modTime, errata) VALUES(NULL,0,0,\'aamp\',\'Loaded 27186468 chains into cavPor3 chain table manually\', NOW(), NULL)\"\
/" loadUp.csh > manualLoadUp.csh
chmod +x manualLoadUp.csh
time nice -n +19 ./manualLoadUp.csh
# [detach screen]
#real    584m4.093s
cd ../../andy/
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit \
  -workhorse=hgwdev -chainMinScore=3000 -chainLinearGap=medium \
  -smallClusterHub=memk -bigClusterHub=swarm -continue=download \
  cavPor3.DEF >& cavPor3.doAfterLoad.log
#real    5m45.122s

# syntenic nets 

screen -r bosTau4
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit \
  -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
  -chainMinScore=3000 -chainLinearGap=medium -syntenicNet \
  -continue=syntenicNet bosTau4.DEF >& bosTau4.syn.log
#real    31m48.545s

# reciprocal best choHof1 and cavPor3
screen -r choHof1
time nice -n +19 doRecipBest.pl -buildDir=/hive/data/genomes/hg19/bed/lastz.choHof1 \
  -workhorse=hgwdev hg19 choHof1 >& choHof1.doRecip.log
#real    367m52.993s

screen -r cavPor3
time nice -n +19 doRecipBest.pl -buildDir=/hive/data/genomes/hg19/bed/lastz.cavPor3 \
  -workhorse=hgwdev hg19 cavPor3 >& cavPor3.doRecip.log
#real    123m3.795s

# reciprocal best small six genome memk run

screen -S recipRun
mkdir recipRun
cd recipRun/
cat > gsub << 'EOF'
#LOOP
./doRecip.sh $(path1) 
#ENDLOOP
'EOF'
cat > doRecip.sh << 'EOF'
#!/bin/csh -ef
set db = $1
/cluster/bin/scripts/doRecipBest.pl -workhorse=`uname -n` -stop=recipBest -buildDir=/hive/data/genomes/hg19/bed/lastz.$db hg19 $db >& $db.recipBest.log
'EOF'
chmod +x doRecip.sh
cat > db.lst << 'EOF'
speTri1
vicPac1
myoLuc1
turTru1
pteVam1
felCat3
EOF
ssh memk
cd /hive/data/genomes/hg19/bed/andy/recipRun
gensub2 db.lst single gsub jobList
para create jobList
para push
# finished overnight
exit # to hgwdev
for log in *.recipBest.log; do 
  db=${log%.recipBest.log}; 
  echo $db;
  doRecipBest.pl -workhorse=hgwdev -continue=download \
    -buildDir=/hive/data/genomes/hg19/bed/lastz.$db \
     hg19 $db >& $db.recipBestDownload.log;
done

# swaps for equCab2, felCat3, bostTau4, cavPor3

cd /hive/data/genomes/hg19/bed/andy
screen -r equCab2
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit -workhorse=kkr14u01 \
  -chainMinScore=3000 -chainLinearGap=medium -swap equCab2.DEF >& equCab2.doSwap.log
# [detach screen]
#real    486m35.206s

screen -r felCat3
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit -workhorse=kkr14u02 \
  -chainMinScore=3000 -chainLinearGap=medium -swap felCat3.DEF >& felCat3.doSwap.log
# [detach screen]
#real    463m5.257s

screen -r bosTau4
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit -workhorse=kkr14u03 \
  -chainMinScore=3000 -chainLinearGap=medium -swap bosTau4.DEF >& bosTau4.doSwap.log
# [detach screen]
#real    391m40.132s

screen -r cavPor3
time nice -n +19 doBlastzChainNet.pl -verbose=2 -noLoadChainSplit -workhorse=hgwdev
  -chainMinScore=3000 -chainLinearGap=medium -swap cavPor3.DEF >& cavPor3.doSwap.log
# [detach screen]
real    192m39.792s

##########################################################################
# LASTZ Venter's Poodle canFamPoodle1 (DONE - 2009-06-05,10 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzCanFamPoodle1.2009-06-05
    cd /hive/data/genomes/hg19/bed/lastzCanFamPoodle1.2009-06-05

    cat << '_EOF_' > DEF
# human vs Venter's poodle

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Dog CanFam2
SEQ2_DIR=/scratch/data/canFamPoodle1/canFamPoodle1.2bit
SEQ2_LEN=/scratch/data/canFamPoodle1/chrom.sizes
SEQ2_CHUNK=40000000
SEQ2_LAP=0
SEQ2_LIMIT=600

BASE=/hive/data/genomes/hg19/bed/lastzCanFamPoodle1.2009-06-05
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl \
        -verbose=2 \
        `pwd`/DEF \
        -noDbNameCheck -noLoadChainSplit \
        -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
        -chainMinScore=3000 -chainLinearGap=medium
    #	real    5162m58.743s
    cat fb.hg19.chainCanFamPoodle1Link.txt 
    #	898034247 bases of 2897316137 (30.995%) in intersection
    #	the original canFam2 measured:
    #	1532073507 bases of 2897316137 (52.879%) in intersection

    time nice -n +19 doRecipBest.pl -buildDir=`pwd` \
	hg19 canFamPoodle1 > rbest.log 2>&1 &
    #	real    811m27.965s

##############################################################################
## 46-Way Multiz (DONE - 2009-06-09,2009-11-10 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/multiz46way
    cd /hive/data/genomes/hg19/bed/multiz46way

    #	starting with the 46way tree created from 44 way tree
    cat << '_EOF_' > 46way.nh
(((((((((((((((((
((hg19:0.006591,panTro2:0.006639):0.002184,gorGor1:0.009411):0.009942,
ponAbe2:0.018342):0.014256,rheMac2:0.036199):0.021496,papHam1:0.04):0.02,
calJac1:0.066389):0.056911,tarSyr1:0.135169):0.011307,
(micMur1:0.091452,otoGar1:0.128984):0.035463):0.015304,
tupBel1:0.183583):0.004688,(((((mm9:0.083220,rn4:0.090564):0.196605,
dipOrd1:0.209532):0.022555,cavPor3:0.223415):0.009828,
speTri1:0.146894):0.025042,
(oryCun2:0.116009,ochPri2:0.198295):0.100037):0.015355):0.020666,
(((vicPac1:0.105252,(turTru1:0.064182,bosTau4:0.121911):0.025111):0.039691,
((equCab2:0.107726,(felCat3:0.097971,canFam2:0.100888):0.049486):0.006252,
(myoLuc1:0.141155,pteVam1:0.111787):0.033187):0.004179):0.011699,
(eriEur1:0.220580,sorAra1:0.266859):0.056117):0.021065):0.023276,
(((loxAfr3:0.083775,proCap1:0.152633):0.026190,echTel1:0.240221):0.049905,
(dasNov2:0.115179,choHof1:0.096272):0.052373):0.006713):0.132748,
macEug1:0.3):0.1,
monDom5:0.325899):0.072430,ornAna1:0.453916):0.109903,
((galGal3:0.166386,taeGut1:0.170717):0.199763,
anoCar1:0.509545):0.108130):0.166150,xenTro2:0.852482):0.300396,
(((tetNig2:0.224774,fr2:0.205294):0.191836,
(gasAcu1:0.313967,oryLat2:0.478451):0.058404):0.322824,
danRer6:0.731166):0.155214):0.511293,petMar1:0.511293);
'_EOF_'
    # << happy emacs

    #	Use this specification in the phyloGif tool:
    #	http://genome.ucsc.edu/cgi-bin/phyloGif
    #	to obtain a gif image for htdocs/images/phylo/hg19_46way.gif

    /cluster/bin/phast/all_dists 46way.nh > 46way.distances.txt
    #	Use this output to create the table below, with this perl script:
    cat << '_EOF_' > sizeStats.pl
#!/usr/bin/env perl

use strict;
use warnings;

open (FH, "grep -y hg19 46way.distances.txt | sort -k3,3n|") or
        die "can not read 46way.distances.txt";

my $count = 0;
while (my $line = <FH>) {
    chomp $line;
    my ($hg19, $D, $dist) = split('\s+', $line);
    my $chain = "chain" . ucfirst($D);
    my $B="/hive/data/genomes/hg19/bed/lastz.$D/fb.hg19." .
        $chain . "Link.txt";
    my $chainLinkMeasure =
        `awk '{print \$5}' ${B} 2> /dev/null | sed -e "s/(//; s/)//"`;
    chomp $chainLinkMeasure;
    $chainLinkMeasure = 0.0 if (length($chainLinkMeasure) < 1);
    $chainLinkMeasure =~ s/\%//;
    my $swapFile="/hive/data/genomes/${D}/bed/blastz.hg19.swap/fb.${D}.chainHg19Link.txt";
    my $swapMeasure = "N/A";
    if ( -s $swapFile ) {
	$swapMeasure =
	    `awk '{print \$5}' ${swapFile} 2> /dev/null | sed -e "s/(//; s/)//"`;
	chomp $swapMeasure;
	$swapMeasure = 0.0 if (length($swapMeasure) < 1);
	$swapMeasure =~ s/\%//;
    }
    my $orgName=
    `hgsql -N -e "select organism from dbDb where name='$D';" hgcentraltest`;
    chomp $orgName;
    if (length($orgName) < 1) {
        $orgName="N/A";
    }
    ++$count;
    if ($swapMeasure eq "N/A") {
	printf "# %02d  %.4f - %s %s\t(%% %.3f) (%s)\n", $count, $dist,
	    $orgName, $D, $chainLinkMeasure, $swapMeasure
    } else {
	printf "# %02d  %.4f - %s %s\t(%% %.3f) (%% %.3f)\n", $count, $dist,
	    $orgName, $D, $chainLinkMeasure, $swapMeasure
    }
}
close (FH);
'_EOF_'
    # << happy emacs
    chmod +x ./sizeStats.pl
    ./sizeStats.pl
#
#	If you can fill in all the numbers in this table, you are ready for
#	the multiple alignment procedure
#
#                         featureBits chainLink measures
#                                        chainOryLat1Link   chain    linearGap
#    distance                      on hg19    on other   minScore
# 01  0.0132 - Chimp panTro2    (% 94.846) (% 94.908)
# 02  0.0182 - Gorilla gorGor1  (% 59.484) (N/A)
# 03  0.0371 - Orangutan ponAbe2        (% 91.350) (% 89.617)
# 04  0.0692 - Rhesus rheMac2   (% 82.744) (% 87.422)
# 05  0.0945 - Baboon papHam1   (% 82.810) (N/A)
# 06  0.1409 - Marmoset calJac1 (% 70.860) (% 71.897)
# 07  0.2665 - Tarsier tarSyr1  (% 47.830) (N/A)
# 08  0.2696 - Mouse lemur micMur1      (% 46.519) (N/A)
# 09  0.3071 - Bushbaby otoGar1 (% 43.644) (N/A)
# 10  0.3343 - Horse equCab2    (% 57.050) (% 66.774)
# 11  0.3416 - TreeShrew tupBel1        (% 36.156) (N/A)
# 12  0.3451 - Dolphin turTru1  (% 48.398) (N/A)
# 13  0.3500 - Squirrel speTri1 (% 35.713) (N/A)
# 14  0.3611 - Alpaca vicPac1   (% 39.399) (N/A)
# 15  0.3620 - Sloth choHof1    (% 34.377) (N/A)
# 16  0.3653 - Megabat pteVam1  (% 45.414) (N/A)
# 17  0.3732 - Elephant loxAfr3 (% 46.636) (% 42.430)
# 18  0.3740 - Cat felCat3      (% 35.713) (% 61.104)
# 19  0.3769 - Dog canFam2      (% 52.879) (% 62.055)
# 20  0.3809 - Armadillo dasNov2        (% 33.543) (N/A)
# 21  0.3941 - Rabbit oryCun2   (% 44.317) (58.405)
# 22  0.3946 - Microbat myoLuc1 (% 33.174) (N/A)
# 23  0.4028 - Cow bosTau4      (% 46.506) (% 50.297)
# 24  0.4363 - Guinea Pig cavPor3       (% 43.680) (N/A)
# 25  0.4421 - Rock hyrax proCap1       (% 30.864) (N/A)
# 26  0.4450 - Kangaroo rat dipOrd1     (% 27.161) (N/A)
# 27  0.4764 - Pika ochPri2     (% 27.768) (N/A)
# 28  0.4811 - Hedgehog eriEur1 (% 19.362) (N/A)
# 29  0.5035 - Tenrec echTel1   (% 23.120) (N/A)
# 30  0.5153 - Mouse mm9        (% 35.299) (% 38.693)
# 31  0.5226 - Rat rn4  (% 32.879) (% 36.860)
# 32  0.5274 - Shrew sorAra1    (% 19.760) (N/A)
# 33  0.6394 - Wallaby macEug1  (% 6.011) (N/A)
# 34  0.7653 - Opossum monDom5  (% 14.358) (N/A)
# 35  0.9657 - Platypus ornAna1 (% 7.627) (% 11.259)
# 36  1.0960 - Chicken galGal3  (% 3.591) (% 8.786)
# 37  1.1003 - Zebra finch taeGut1      (% 3.496) (% 7.795)
# 38  1.2394 - Lizard anoCar1   (% 3.591) (% 5.146)
# 39  1.6403 - X. tropicalis xenTro2    (% 3.176) (% 6.773)
# 40  1.9387 - Stickleback gasAcu1      (% 1.916) (% 11.175)
# 41  1.9634 - Fugu fr2 (% 1.702) (% 10.929)
# 42  1.9746 - Zebrafish danRer6        (% 3.051) (% 6.399)
# 43  1.9829 - Tetraodon tetNig2        (% 1.712) (% 14.194)
# 44  2.1031 - Medaka oryLat2   (% 1.849) (% 6.705)
# 45  2.1108 - Lamprey petMar1  (% 1.082) (% 3.200)

    # create species list and stripped down tree for autoMZ
    sed 's/[a-z][a-z]*_//g; s/:[0-9\.][0-9\.]*//g; s/;//; /^ *$/d' \
	46way.nh > tmp.nh
    echo `cat tmp.nh` > tree-commas.nh
    echo `cat tree-commas.nh` | sed 's/ //g; s/,/ /g' > tree.nh
    sed 's/[()]//g; s/,/ /g' tree.nh > species.list

    cd /hive/data/genomes/hg19/bed/multiz46way
    #	bash shell syntax here ...
    export H=/hive/data/genomes/hg19/bed
    mkdir mafLinks
    for G in `sed -e "s/hg19 //" species.list`
    do
	mkdir mafLinks/$G
	if [ -s ${H}/lastz.${G}/mafRBestNet/chr1.maf.gz ]; then
	    echo "$G - recipBest"
	    ln -s ${H}/lastz.$G/mafRBestNet/*.maf.gz ./mafLinks/$G
	else
	    if [ -s ${H}/lastz.${G}/mafSynNet/chr1.maf.gz ]; then
		echo "$G - synNet"
		ln -s ${H}/lastz.$G/mafSynNet/*.maf.gz ./mafLinks/$G
	    else
		if [ -s ${H}/lastz.${G}/mafNet/chr1.maf.gz ]; then
		    echo "$G - mafNet"
		    ln -s ${H}/lastz.$G/mafNet/*.maf.gz ./mafLinks/$G
		else
		    echo "missing directory lastz.${G}/*Net"
		fi
	    fi
	fi
    done

    #	verify the alignment type is correct:
    for D in `cat /hive/users/hiram/bigWayHg19/ordered.list`
do
    ls -l mafLinks/$D/chr1.maf.gz | awk '{print $NF}'
done
    #	compare to the list at:
    #	http://genomewiki.ucsc.edu/index.php/Hg19_Genome_size_statistics

    #	need to split these things up into smaller pieces for
    #	efficient kluster run.
    cd /hive/data/genomes/hg19/bed/multiz46way
    mkdir mafSplit
    cd mafSplit
    #	mafSplitPos splits on gaps or repeat areas that will not have
    #	any chains, approx 5 Mbp intervals, gaps at least 10,000
    mafSplitPos -minGap=10000 hg19 5stdout | sort -u \
	| sort -k1,1 -k2,2n > mafSplit.bed
    #	There is a splitRegions.pl script here (copied from previous 44way)
    #	that can create a custom track from this mafSplit.bed file.
    #	Take a look at that in the browser and see if it looks OK,
    #	check the number of sections on each chrom to verify none are
    #	too large.  Despite the claim above, it does appear that some
    #	areas are split where actual chains exist.

    #	run a small kluster job to split them all
    ssh memk
    cd /hive/data/genomes/hg19/bed/multiz46way/mafSplit
    cat << '_EOF_' > runOne
#!/bin/csh -ef
set G = $1
set C = $2
mkdir -p $G
pushd $G > /dev/null
if ( -s ../../mafLinks/${G}/${C}.maf.gz ) then
    rm -f hg19_${C}.*.maf
    mafSplit ../mafSplit.bed hg19_ ../../mafLinks/${G}/${C}.maf.gz
    gzip hg19_${C}.*.maf
else
    touch hg19_${C}.00.maf
    gzip hg19_${C}.00.maf
endif
popd > /dev/null
'_EOF_'
    # << happy emacs
    chmod +x runOne

    cat << '_EOF_' > template
#LOOP
runOne $(root1) $(root2) {check out line $(root1)/hg19_$(root2).00.maf}
#ENDLOOP
'_EOF_'
    # << happy emacs

    for G in `sed -e "s/hg19 //" ../species.list`
do
    echo $G
done > species.list
    cut -f 1 ../../../chrom.sizes > chr.list

    gensub2 species.list chr.list template jobList
    para -ram=8g create jobList
    para try ... check ... push ... etc...
# Completed: 4185 of 4185 jobs
# CPU time in finished jobs:      25547s     425.78m     7.10h    0.30d  0.001 y
# IO & Wait Time:                268664s    4477.73m    74.63h    3.11d  0.009 y
# Average job time:                  70s       1.17m     0.02h    0.00d
# Longest finished job:            1234s      20.57m     0.34h    0.01d
# Submission to last job:          3048s      50.80m     0.85h    0.04d

    # the autoMultiz cluster run
    ssh swarm
    cd /hive/data/genomes/hg19/bed/multiz46way/

    mkdir splitRun
    cd splitRun
    mkdir maf run
    cd run
    mkdir penn
    cp -p /cluster/bin/penn/multiz.2008-11-25/multiz penn 
    cp -p /cluster/bin/penn/multiz.2008-11-25/maf_project penn 
    cp -p /cluster/bin/penn/multiz.2008-11-25/autoMZ penn 

    #	set the db and pairs directories here
    cat > autoMultiz.csh << '_EOF_'
#!/bin/csh -ef
set db = hg19
set c = $1
set result = $2
set run = `/bin/pwd`
set tmp = /scratch/tmp/$db/multiz.$c
set pairs = /hive/data/genomes/hg19/bed/multiz46way/mafSplit
/bin/rm -fr $tmp
/bin/mkdir -p $tmp
/bin/cp -p ../../tree.nh ../../species.list $tmp
pushd $tmp > /dev/null
foreach s (`/bin/sed -e "s/ $db//" species.list`)
    set in = $pairs/$s/$c.maf
    set out = $db.$s.sing.maf
    if (-e $in.gz) then
        /bin/zcat $in.gz > $out
	if (! -s $out) then
	    echo "##maf version=1 scoring=autoMZ" > $out
	endif
    else if (-e $in) then
        /bin/ln -s $in $out
    else
        echo "##maf version=1 scoring=autoMZ" > $out
    endif
end
set path = ($run/penn $path); rehash
$run/penn/autoMZ + T=$tmp E=$db "`cat tree.nh`" $db.*.sing.maf $c.maf \
	> /dev/null
popd > /dev/null
/bin/rm -f $result
/bin/cp -p $tmp/$c.maf $result
/bin/rm -fr $tmp
/bin/rmdir --ignore-fail-on-non-empty /scratch/tmp/$db
'_EOF_'
# << happy emacs
    chmod +x autoMultiz.csh

    cat  << '_EOF_' > template
#LOOP
./autoMultiz.csh $(root1) {check out line+ /hive/data/genomes/hg19/bed/multiz46way/splitRun/maf/$(root1).maf}
#ENDLOOP
'_EOF_'
# << happy emacs

    find ../../mafSplit -type f | grep hg19_ | xargs -L 1 basename \
	| sed -e "s/.gz//" | sort -u > chr.part.list
    gensub2 chr.part.list single template jobList
    para -ram=8g create jobList
    #	initial run experience suggest some of the big jobs reach 8 Gb
    #	of memory usage, so, tell parasol to limit the number of jobs per
    #	node to avoid thrashing
    para -ram=8g try
    para -ram=8g push
# Completed: 504 of 504 jobs
# CPU time in finished jobs:    1342039s   22367.32m   372.79h   15.53d  0.043 y
# IO & Wait Time:                 63835s    1063.91m    17.73h    0.74d  0.002 y
# Average job time:                2789s      46.49m     0.77h    0.03d
# Longest finished job:           12625s     210.42m     3.51h    0.15d
# Submission to last job:         15300s     255.00m     4.25h    0.18d

    # put the split maf results back together into a single maf file
    #	eliminate duplicate comments
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/multiz46way/splitRun
    mkdir ../maf
    #	the sed edits take out partitioning name information from the comments
    #	so the multiple parts will condense to smaller number of lines
    #	this takes almost 2 hours of time, resulting in a bit over 150 Gb,
    #	almost all chrom files over 1 Gb, up to almost 10 Gb for chr2
    #	HOWEVER, this is actually not necessary to maintain these comments,
    #	they are lost during the mafAddIRows

    cat << '_EOF_' >> runOne
#!/bin/csh -fe
set C = $1
if ( -s ../maf/${C}.maf.gz ) then
    rm -f ../maf/${C}.maf.gz
endif
head -q -n 1 maf/hg19_${C}.*.maf | sort -u > ../maf/${C}.maf
grep -h "^#" maf/hg19_${C}.*.maf | egrep -v "maf version=1|eof maf" | \
    sed -e "s#${C}.[0-9][0-9]*#${C}#g; s#_MZ_[^ ]* # #g;" \
        | sort -u >> ../maf/${C}.maf 
grep -h -v "^#" `ls maf/hg19_${C}.*.maf | sort -t. -k2,2n` >> ../maf/${C}.maf
tail -q -n 1 maf/hg19_${C}.*.maf | sort -u >> ../maf/${C}.maf
'_EOF_'
    # << happy emacs
    chmod +x runOne

    cat << '_EOF_' >> template
#LOOP
runOne $(root1) {check out exists+ ../maf/$(root1).maf}
#ENDLOOP
'_EOF_'
    # << happy emacs

    cut -f1 ../../../chrom.sizes > chr.list
    ssh encodek
    cd /hive/data/genomes/hg19/bed/multiz46way/splitRun
    gensub2 chr.list single template jobList
    para create jobList
    para try ... check ... push ... etc ...
# Completed: 92 of 93 jobs
# Crashed: 1 jobs
# CPU time in finished jobs:        412s       6.86m     0.11h    0.00d  0.000 y
# IO & Wait Time:                 21187s     353.12m     5.89h    0.25d  0.001 y
# Average job time:                 235s       3.91m     0.07h    0.00d
# Longest finished job:            1529s      25.48m     0.42h    0.02d
# Submission to last job:          1542s      25.70m     0.43h    0.02d

    #	one of the results is completely empty, the grep for results failed
    #	this file ../maf/chrUn_gl000226.maf only has header comments, no result

    # load tables for a look
    ssh hgwdev
    mkdir -p /gbdb/hg19/multiz46way/maf
    cd /hive/data/genomes/hg19/bed/multiz46way/maf
    ln -s `pwd`/*.maf /gbdb/hg19/multiz46way/maf

    # this generates an immense multiz46way.tab file in the directory
    #	where it is running.  Best to run this over in scratch.
    cd /data/tmp
    time nice -n +19 hgLoadMaf \
	-pathPrefix=/gbdb/hg19/multiz46way/maf hg19 multiz46way
    #	Loaded 33558634 mafs in 93 files from /gbdb/hg19/multiz46way/maf
    #	real    512m8.053s

    # load summary table
    time nice -n +19 cat /gbdb/hg19/multiz46way/maf/*.maf \
	| $HOME/bin/$MACHTYPE/hgLoadMafSummary hg19 -minSize=30000 -verbose=2 \
		-mergeGap=1500 -maxSize=200000  multiz46waySummary stdin
    #	real    92m30.700s
# flushSummaryBlocks: output 45 blocks
# Created 8766427 summary blocks from 645238409 components and
#	33558634 mafs from stdin
# blocks too small to be used: 29456
# Loading into hg19 table multiz46waySummary...

    # Gap Annotation
    # prepare bed files with gap info
    mkdir /hive/data/genomes/hg19/bed/multiz46way/anno
    cd /hive/data/genomes/hg19/bed/multiz46way/anno
    mkdir maf run

    #	most of these will already exist from previous multiple alignments
    #	remove the echo from in front of the twoBitInfo command to get them
    #	to run if this loop appears to be correct
    for DB in `cat ../species.list`
do
    CDIR="/hive/data/genomes/${DB}"
    if [ ! -f ${CDIR}/${DB}.N.bed ]; then
	echo "creating ${DB}.N.bed"
	echo twoBitInfo -nBed ${CDIR}/${DB}.2bit ${CDIR}/${DB}.N.bed
    else
	ls -og ${CDIR}/${DB}.N.bed
    fi
done

    cd run
    rm -f nBeds sizes
    for DB in `sed -e "s/hg19 //" ../../species.list`
do
    echo "${DB} "
    ln -s  /hive/data/genomes/${DB}/${DB}.N.bed ${DB}.bed
    echo ${DB}.bed  >> nBeds
    ln -s  /hive/data/genomes/${DB}/chrom.sizes ${DB}.len
    echo ${DB}.len  >> sizes
done

    #	the annotation step requires large memory, run on memk nodes
    ssh memk
    cd /hive/data/genomes/hg19/bed/multiz46way/anno/run
    ls ../../maf | sed -e "s/.maf//" > chr.list
    cat << '_EOF_' > template
#LOOP
./anno.csh $(root1) {check out line+ ../maf/$(root1).maf}
#ENDLOOP
'_EOF_'
    # << happy emacs

    cat << '_EOF_' > anno.csh
#!/bin/csh -fe

set inMaf = ../../maf/$1.maf
set outMaf = ../maf/$1.maf
rm -f $outMaf
mafAddIRows -nBeds=nBeds $inMaf /hive/data/genomes/hg19/hg19.2bit $outMaf
'_EOF_'
    # << happy emacs
    chmod +x anno.csh

    gensub2 chr.list single template jobList
    para -ram=30g create jobList
    #	specify lots of ram to get one job per node
    para -ram=30g push
    #	
# Completed: 93 of 93 jobs
# CPU time in finished jobs:      10371s     172.85m     2.88h    0.12d  0.000 y
# IO & Wait Time:                  3365s      56.09m     0.93h    0.04d  0.000 y
# Average job time:                 148s       2.46m     0.04h    0.00d
# Longest finished job:            1153s      19.22m     0.32h    0.01d
# Submission to last job:          7402s     123.37m     2.06h    0.09d

    ssh hgwdev
    rm -fr /gbdb/hg19/multiz46way/maf
    mkdir /gbdb/hg19/multiz46way/maf
    cd /hive/data/genomes/hg19/bed/multiz46way/anno/maf
    ln -s `pwd`/*.maf /gbdb/hg19/multiz46way/maf/
    #	by loading this into the table multiz46way, it will replace the
    #	previously loaded table with the unannotated mafs
    #	huge temp files are made, do them on local disk
    cd /data/tmp
    time nice -n +19 hgLoadMaf \
	-pathPrefix=/gbdb/hg19/multiz46way/maf hg19 multiz46way
    #	real    113m11.709s
    #	Loaded 33612571 mafs in 93 files from /gbdb/hg19/multiz46way/maf
XXX - done to here

    time nice -n +19 cat /gbdb/hg19/multiz46way/maf/*.maf \
	| hgLoadMafSummary hg19 -minSize=30000 -mergeGap=1500 \
                 -maxSize=200000  multiz46waySummary stdin
    #	with the quality annotated mafs, and mem interference on hgwdev:
    #	Created 8514381 summary blocks from 600504256 components \
    #	and 33320838 mafs from stdin
    #	real    169m56.936s

    #	with the Irow annotations after the multiz fix:
    #	Created 8514380 summary blocks from 600499937
    #		components and 33298894 mafs from stdin
    #	real    184m42.893s
    #	user    70m44.431s
    #	sys     8m7.970s

    #	Created 8514078 summary blocks from 604683213 components
    #	and 35125649 mafs from stdin
    #	real    130m55.115s
    #	user    71m37.409s
    #	sys     8m5.110s

    #	by loading this into the table multiz46waySummary, it will replace
    #	the previously loaded table with the unannotated mafs
    #	remove the multiz46way*.tab files in this /data/tmp directory
# -rw-rw-r--   1 1949221892 Nov 15 14:04 multiz46way.tab
# -rw-rw-r--   1  417994189 Nov 15 20:57 multiz46waySummary.tab
    wc -l multiz46way*.tab
    #	33964377 multiz46way.tab
    #	 8514078 multiz46waySummary.tab
    #	42478455 total
    rm multiz46way*.tab

    # create some downloads
    mkdir -p /hive/data/genomes/hg19/bed/multiz46way/download/maf
    cd /hive/data/genomes/hg19/bed/multiz46way/download/maf
    time cp -p ../../anno/maf/chr*.maf .
    #	real    72m46.514s
    #	user    0m1.293s
    #	sys     5m15.981s
    time gzip --rsyncable *.maf
    time gzip --rsyncable *.maf
    #	real    185m37.884s
    #	user    179m51.161s
    #	sys     3m48.016s
    time md5sum *.gz > md5sum.txt
    #	real    3m59.009s
    #	user    1m19.338s
    #	sys     0m18.976s

##############################################################################
# LASTZ Sea Hare aplCal1 (STARTING - 2009-06-08 - Galt)
# To Do #1813 remove aplCal1 <-> hg19 chain/nets (2011-10-07 Chin)
# However, only tables and download files are physically drop/removed
# All data created stay on hive.
    mkdir /hive/data/genomes/hg19/bed/lastzAplCal1.2009-06-08
    cd /hive/data/genomes/hg19/bed/lastzAplCal1.2009-06-08

    cat << '_EOF_' > DEF
# Human vs. Sea Hare
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_M=50
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=100000000
SEQ1_LAP=10000
SEQ2_LIMIT=5

# QUERY: Sea Hare aplCal1
SEQ2_DIR=/scratch/data/aplCal1/aplCal1.2bit
SEQ2_LEN=/scratch/data/aplCal1/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LIMIT=300
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzAplCal1.2009-06-08
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    #   (NOTE I SHOULD NOT HAVE USED  -qRepeats=windowmaskerSdust)
    screen
    time nice +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=pk \
	>& do.log &
    #	real about one hour but one job hung

    # resuming from failure 
    # edited loadUp.csh, commenting out the first completed step
    # and removing the unneeded -qRepeats=windowmaskerSdust
    # from the next step, now run it to complete the load step.
    /hive/data/genomes/hg19/bed/lastzAplCal1.2009-06-08/axtChain/loadUp.csh \ 
        >& continue-loadUp.log&

    # continue from step 'download'
    time nice +19 doBlastzChainNet.pl -verbose=2 \
        `pwd`/DEF \
        -noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
        -workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=pk \
        -continue download \
        >& continue-download.log &
 
    cat fb.hg19.chainAplCal1Link.txt 
    #   19675762 bases of 2897316137 (0.679%) in intersection

    #	running the swap - DONE - 2009-06-02
    #   (NOTE I SHOULD NOT HAVE USED  -qRepeats=windowmaskerSdust)
    mkdir /hive/data/genomes/aplCal1/bed/blastz.hg19.swap
    cd /hive/data/genomes/aplCal1/bed/blastz.hg19.swap
    time nice +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzAplCal1.2009-06-08/DEF \
	-qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=pk \
	-swap >& swap.log &
    #	real  time not long

    # resuming from failure
    # edited loadUp.csh, commenting out the first completed step
    # and removing the unneeded -tRepeats=windowmaskerSdust
    # from the next step, now run it to complete the load step.
    /hive/data/genomes/aplCal1/bed/blastz.hg19.swap/axtChain/loadUp.csh \
        >& continue-loadUp.log&

    time nice +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzAplCal1.2009-06-08/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=pk \
        -continue download \
	-swap >& continue-download.log &

    cat fb.aplCal1.chainHg19Link.txt 
    #   14163455 bases of 619228098 (2.287%) in intersection

#########################################################################
# EXONIPHY Hg19, lifted from hg18 (DONE - 2009-06-19 - Hiram)
#	needed for uscsGenes11 building
    # create a syntenic liftOver chain file
    cd /hive/data/genomes/hg18/bed/blat.hg19.2009-03-06
    time nice -n +19 netSyntenic run.chain/hg18.hg19.noClass.net.gz stdout \
	| netFilter -syn stdin | netChainSubset -verbose=0 stdin \
		run.chain/hg18.hg19.all.chain.gz stdout \
	| chainStitchId stdin stdout | gzip -c > hg18.hg19.syn.chain.gz
    #	memory usage 55492608, utime 3 s/100, stime 3
    #	real    2m35.613s

    #	real    5m55.575s
    #	slightly smaller than the ordinary liftOver chain file:
# -rw-rw-r-- 1 137245 Mar  6 17:37 hg18ToHg19.over.chain.gz
# -rw-rw-r-- 1  96115 Jun 19 14:30 hg18.hg19.syn.chain.gz

    # exoniphyHg19.gp is prepared as follows
    mkdir /cluster/data/hg19/bed/exoniphy
    cd /cluster/data/hg19/bed/exoniphy
    hgsql hg18 -e "select * from exoniphy" -N > exoniphyHg18.gp
    time nice -n +19 liftOver -genePred exoniphyHg18.gp \
      /hive/data/genomes/hg18/bed/blat.hg19.2009-03-06/hg18.hg19.syn.chain.gz \
	    exoniphyHg19.gp unmapped
    wc -l *
    #	178162 exoniphyHg18.gp
    #	178109 exoniphyHg19.gp
    #	   106 unmapped

    mkdir dump
    cd dump
    hgsqldump --all -c --tab=. hg18 exoniphy
    cd ..
    chmod 775 dump
    hgsql hg19 < dump/exoniphy.sql
    hgsql hg19 \
-e "load data local infile \"exoniphyHg19.gp\" into table exoniphy;"
    nice -n +19 featureBits hg19 exoniphy
    #	27421336 bases of 2897316137 (0.946%) in intersection
    nice -n +19 featureBits hg18 exoniphy
    #	27475705 bases of 2881515245 (0.954%) in intersection

#########################################################################
# BIOCYCTABLES NEEDED BY hgGene (DONE - 2009-06-22 - Hiram)

# First register with BioCyc to download their HumanCyc database
# The site will email you the URL for download.  Beware, they supply
#	a URL to a directory chock a block full of data, almost 7 Gb,
#	you only need one file

    mkdir /hive/data/outside/bioCyc/090623
    cd /hive/data/outside/bioCyc/090623

    mkdir download
    cd download
    wget --timestamping --no-directories --recursive \
	"http://bioinformatics.ai.sri.com/ecocyc/dist/flatfiles-52983746/humancyc-flatfiles.tar.Z"
    tar xvzf humancyc-flatfiles.tar.Z

    mkdir /hive/data/genomes/hg19/bed/bioCyc
    cd /hive/data/genomes/hg19/bed/bioCyc
    #	clean the headers from these files
    grep -E -v "^#|^UNIQUE-ID" /hive/data/outside/bioCyc/090623/genes.col \
	> genes.tab
    #	this file isn't consistent in its number of columns
    grep -E -v "^#|^UNIQUE-ID" /hive/data/outside/bioCyc/090623/pathways.col \
| awk -F'\t' '{if (140 == NF) { printf "%s\t\t\n", $0; } else { print $0}}' \
	> pathways.tab

    hgsql hg19 -e 'create database bioCyc090623'

    hgLoadSqlTab bioCyc090623 genes ~/src/hg/lib/bioCycGenes.sql ./genes.tab
    hgLoadSqlTab bioCyc090623 pathways ~/src/hg/lib/bioCycPathways.sql ./pathways.tab

# Create bioCycMapDesc.tab
    hgsql bioCyc090623 -N \
	-e 'select UNIQUE_ID, NAME from pathways' | sort -u >  bioCycMapDesc.tab
XXX see alternative below

    #	this kgBioCyc0 thing needs kgXref and other UCSC gene tables to work
# Create bioCycPathway.tab
    kgBioCyc0 bioCyc090623 hg19 hg19

    hgLoadSqlTab hg19 bioCycPathway ~/kent/src/hg/lib/bioCycPathway.sql ./bioCycPathway.tab
    hgLoadSqlTab hg19 bioCycMapDesc ~/kent/src/hg/lib/bioCycMapDesc.sql ./bioCycMapDesc.tab

XXX maybe instead do this in the gene build procedure
    # from the UCSC genes build procedure
# Do BioCyc Pathways build
    mkdir $dir/bioCyc
    cd $dir/bioCyc
    grep -v '^#' $bioCycPathways > pathways.tab
    grep -v '^#' $bioCycGenes > genes.tab
    kgBioCyc1 genes.tab pathways.tab $db bioCycPathway.tab bioCycMapDesc.tab
    hgLoadSqlTab $tempDb bioCycPathway ~/kent/src/hg/lib/bioCycPathway.sql ./bioCycPathway.tab
    hgLoadSqlTab $tempDb bioCycMapDesc ~/kent/src/hg/lib/bioCycMapDesc.sql ./bioCycMapDesc.tab

##############################################################################
nscanGene (2009-06-22 markd)
   # nscanGene track from WUSTL
   cd /cluster/data/hg19/bed/nscan
   wget http://mblab.wustl.edu/~jeltje/hg19_tracks/hg19.updated.gtf
   wget http://mblab.wustl.edu/~jeltje/hg19_tracks/hg19.readme
   wget -r -np -l 1 http://mblab.wustl.edu/~jeltje/hg19_tracks/hg19_proteins
   bzip2 hg19.updated.gtf hg19_proteins/*.fa

   # load track
   gtfToGenePred -genePredExt hg19.updated.gtf.bz2 stdout| hgLoadGenePred -genePredExt hg19 nscanGene stdin
   bzcat hg19_proteins/chr*.fa.bz2 | hgPepPred hg19 generic nscanPep stdin
   rm *.tab

   # validate same number of transcripts and peptides are loaded
   hgsql -Ne 'select count(*) from nscanGene' hg19
   hgsql -Ne 'select count(*) from nscanPep' hg19

   # validate search expression
   hgc-sql -Ne 'select name from nscanGene' hg19 | egrep -v -e '^chr[0-9a-zA-Z_]+\.([0-9]+|pasa)((\.[0-9a-z]+)?\.[0-9a-z]+)?$' |wc -l

#########################################################################
# Phylogenetic tree from 46-way for chrX  (DONE - 2009-10-26 - Hiram)
# 	We need two trees, one for chrX only, and a second for all other chroms
    mkdir /hive/data/genomes/hg19/bed/multiz46way/4dX
    cd /hive/data/genomes/hg19/bed/multiz46way/4dX

    hgsql hg19 -Ne \
    "select * from refGene,refSeqStatus where refGene.name=refSeqStatus.mrnaAcc and refSeqStatus.status='Reviewed' and mol='mRNA' and refGene.chrom='chrX'" \
	| cut -f 2-20 > refSeqReviewed.gp
    wc -l refSeqReviewed.gp
    # 727 refSeqReviewed.gp
    genePredSingleCover refSeqReviewed.gp stdout | sort > refSeqReviewedNR.gp
    wc -l refSeqReviewedNR.gp
    # 401 refSeqReviewed.gp

    ssh memk
    mkdir /hive/data/genomes/hg19/bed/multiz46way/4dX/run
    cd /hive/data/genomes/hg19/bed/multiz46way/4dX/run
    mkdir ../mfa

# whole chrom mafs version, using new version of 
# uses memory-efficient version of phast, from Melissa Hubisz at Cornell
#	mjhubisz at gmail.com

    cat << '_EOF_' > 4dX.csh
#!/bin/csh -fe
set PHASTBIN = /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin
set r = "/hive/data/genomes/hg19/bed/multiz46way"
set c = $1
set infile = $r/maf/$2
set outfile = $3
cd /scratch/tmp
# 'clean' maf
perl -wpe 's/^s ([^.]+)\.\S+/s $1/' $infile > $c.maf    
awk -v C=$c '$2 == C {print}' $r/4dX/refSeqReviewedNR.gp > $c.gp
set PHASTBIN = /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin
$PHASTBIN/msa_view --4d --features $c.gp --do-cats 3 -i MAF $c.maf -o SS > $c.ss
$PHASTBIN/msa_view -i SS --tuple-size 1 $c.ss > $r/4dX/$outfile
rm -f $c.gp $c.maf $c.ss
'_EOF_'
    # << happy emacs
    chmod +x 4dX.csh

    ls -1S /hive/data/genomes/hg19/bed/multiz46way/maf/chrX.maf | \
        egrep -E -v "chrM|chrUn|random|_hap" | sed -e "s#.*multiz46way/maf/##" \
	> maf.list

    cat << '_EOF_' > template
#LOOP
4dX.csh $(root1) $(path1) {check out line+ mfa/$(root1).mfa}
#ENDLOOP
'_EOF_'
    # << happy emacs

    gensub2 maf.list single template stdout | tac > jobList
    #	run this one job on hgwdev, takes a few minutes:
    ./4dX.csh chrX chrX.maf mfa/chrX.mfa
    #	not sure what these warnings are about:
# WARNING: ignoring out-of-range feature
# chrX    genepred        CDS     1       -1      .       +       2       transcript_id "NM_000475"
# WARNING: ignoring out-of-range feature
# chrX    genepred        CDS     1       -1      .       +       2       transcript_id "NM_005365.2"

    # combine mfa files
    cd ..
    sed -e "s/ /,/g" ../species.list > species.lst
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/msa_view \
	--aggregate `cat species.lst` mfa/*.mfa | sed s/"> "/">"/ > 4dX.chrX.mfa

XXXX ! 2010-12-29 - There is an error in the awk below.
XXXX It ends up only working on the first file mfa/chr1.mfa
XXXX with the result in placentals.mfa only from chr1.mfa
    sed -e 's/,macEug1.*//' species.lst > placentals.lst
    awk '
BEGIN { good = 1 }
{
    if (match($0, "^> macEug1")) { good = 0 }
    if (good) {print}
}
' mfa/*.mfa > placentals.mfa
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/msa_view \
	--aggregate `cat placentals.lst` placentals.mfa | sed s/"> "/">"/ \
	> 4dX.placentals.mfa

XXXX ! 2010-12-29 - There is an error in the awk below.
XXXX It ends up only working on the first file mfa/chr1.mfa
XXXX with the result in primates.mfa only from chr1.mfa
    sed -e 's/,tupBel1.*//' species.lst > primates.lst
    awk '
BEGIN { good = 1 }
{
    if (match($0, "^> tupBel1")) { good = 0 }
    if (good) {print}
}
' mfa/*.mfa > primates.mfa
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/msa_view \
	--aggregate `cat primates.lst` primates.mfa | sed -e "s/> />/" \
	> 4dX.primates.mfa

    # use phyloFit to create tree model (output is phyloFit.mod)
    time /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/phyloFit \
	--EM --precision MED --msa-format FASTA --subst-mod REV \
	--tree ../tree-commas.nh 4dX.chrX.mfa
    #	real    0.54.139s
    mv phyloFit.mod phyloFit.chrX.mod

    grep TREE phyloFit.chrX.mod | sed 's/TREE\:\ //' > tree_4d.chrX.46way.nh

    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/tree_doctor \
        --no-branchlen --prune-all-but=`cat primates.lst` ../tree-commas.nh \
                > tree_commas.primates.nh
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/tree_doctor \
        --no-branchlen --prune-all-but=`cat placentals.lst` ../tree-commas.nh \
                > tree_commas.placentals.nh
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/phyloFit \
	--EM --precision MED --msa-format FASTA --subst-mod REV \
	--tree tree_commas.primates.nh 4dX.primates.mfa
    mv phyloFit.mod phyloFit.chrX.primates.mod
    grep TREE phyloFit.chrX.primates.mod | sed 's/TREE\:\ //' \
	> tree_4d.chrX.primates.46way.nh
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/phyloFit \
	--EM --precision MED --msa-format FASTA --subst-mod REV \
	--tree tree_commas.placentals.nh 4dX.placentals.mfa
    mv phyloFit.mod phyloFit.chrX.placentals.mod
    grep TREE phyloFit.chrX.placentals.mod | sed 's/TREE\:\ //' \
	> tree_4d.chrX.placentals.46way.nh

#########################################################################
# Phylogenetic tree from 46-way for non-chrX  (DONE - 2009-10-27 - Hiram)
# 	We need two trees, one for chrX only, and a second for all other chroms
    mkdir /hive/data/genomes/hg19/bed/multiz46way/4dNoX
    cd /hive/data/genomes/hg19/bed/multiz46way/4dNoX

    hgsql hg19 -Ne \
    "select * from refGene,refSeqStatus where refGene.name=refSeqStatus.mrnaAcc and refSeqStatus.status='Reviewed' and mol='mRNA'" \
	| cut -f 2-20 | egrep -E -v "chrM|chrUn|random|_hap|chrX" \
	> refSeqReviewed.gp
    wc -l refSeqReviewed.gp
    # 12977 refSeqReviewed.gp
    genePredSingleCover refSeqReviewed.gp stdout | sort > refSeqReviewedNR.gp
    wc -l refSeqReviewedNR.gp
    # 7252 refSeqReviewed.gp

    ssh memk
    mkdir /hive/data/genomes/hg19/bed/multiz46way/4dNoX/run
    cd /hive/data/genomes/hg19/bed/multiz46way/4dNoX/run
    mkdir ../mfa

# whole chrom mafs version, using new version of 
# uses memory-efficient version of phast, from Melissa Hubisz at Cornell
#	mjhubisz at gmail.com

    cat << '_EOF_' > 4dNoX.csh
#!/bin/csh -fe
set PHASTBIN = /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin
set r = "/hive/data/genomes/hg19/bed/multiz46way"
set c = $1
set infile = $r/maf/$2
set outfile = $3
cd /scratch/tmp
# 'clean' maf
perl -wpe 's/^s ([^.]+)\.\S+/s $1/' $infile > $c.maf    
awk -v C=$c '$2 == C {print}' $r/4dNoX/refSeqReviewedNR.gp > $c.gp
set PHASTBIN = /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin
$PHASTBIN/msa_view --4d --features $c.gp --do-cats 3 -i MAF $c.maf -o SS > $c.ss
$PHASTBIN/msa_view -i SS --tuple-size 1 $c.ss > $r/4dNoX/$outfile
rm -f $c.gp $c.maf $c.ss
'_EOF_'
    # << happy emacs
    chmod +x 4dNoX.csh

    ls -1S /hive/data/genomes/hg19/bed/multiz46way/maf/chr*.maf | \
        egrep -E -v "chrM|chrUn|random|_hap|chrX" \
	| sed -e "s#.*multiz46way/maf/##" \
	> maf.list

    cat << '_EOF_' > template
#LOOP
4dNoX.csh $(root1) $(path1) {check out line+ ../mfa/$(root1).mfa}
#ENDLOOP
'_EOF_'
    # << happy emacs

    gensub2 maf.list single template stdout | tac > jobList
    para try ... check ... push ... etc
    para time
# Completed: 23 of 23 jobs
# CPU time in finished jobs:       9032s     150.53m     2.51h    0.10d  0.000 y
# IO & Wait Time:                   672s      11.21m     0.19h    0.01d  0.000 y
# Average job time:                 422s       7.03m     0.12h    0.00d
# Longest finished job:             860s      14.33m     0.24h    0.01d
# Submission to last job:          1210s      20.17m     0.34h    0.01d

    # combine mfa files
    cd ..
    sed -e "s/ /,/g" ../species.list > species.lst
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/msa_view \
	--aggregate `cat species.lst` mfa/*.mfa | sed s/"> "/">"/ \
	> 4dNoX.all.mfa

    sed -e 's/,macEug1.*//' species.lst > placentals.lst
    awk '
BEGIN { good = 1 }
{
    if (match($0, "^> macEug1")) { good = 0 }
    if (good) {print}
}
' mfa/*.mfa > placentals.mfa
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/msa_view \
	--aggregate `cat placentals.lst` placentals.mfa | sed s/"> "/">"/ \
	> 4dNoX.placentals.mfa

    sed -e 's/,tupBel1.*//' species.lst > primates.lst
    awk '
BEGIN { good = 1 }
{
    if (match($0, "^> tupBel1")) { good = 0 }
    if (good) {print}
}
' mfa/*.mfa > primates.mfa
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/msa_view \
	--aggregate `cat primates.lst` primates.mfa | sed -e "s/> />/" \
	> 4dNoX.primates.mfa


    # use phyloFit to create tree model (output is phyloFit.mod)
    time /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/phyloFit \
	--EM --precision MED --msa-format FASTA --subst-mod REV \
	--tree ../tree-commas.nh 4dNoX.all.mfa
XXX - running Tue Oct 27 13:21:49 PDT 2009
    #	about 40 minutes
    mv phyloFit.mod phyloFit.NoChrX.mod

    grep TREE phyloFit.chrX.mod | sed 's/TREE\:\ //' > tree_4d.chrX.46way.nh

    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/tree_doctor \
        --no-branchlen --prune-all-but=`cat primates.lst` ../tree-commas.nh \
                > tree_commas.primates.nh
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/tree_doctor \
        --no-branchlen --prune-all-but=`cat placentals.lst` ../tree-commas.nh \
                > tree_commas.placentals.nh
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/phyloFit \
	--EM --precision MED --msa-format FASTA --subst-mod REV \
	--tree tree_commas.primates.nh 4dNoX.primates.mfa
    mv phyloFit.mod phyloFit.NoChrX.primates.mod
    grep TREE phyloFit.NoChrX.primates.mod | sed 's/TREE\:\ //' \
	> tree_4d.NoChrX.primates.46way.nh
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/phyloFit \
	--EM --precision MED --msa-format FASTA --subst-mod REV \
	--tree tree_commas.placentals.nh 4dNoX.placentals.mfa
    mv phyloFit.mod phyloFit.NoChrX.placentals.mod
    grep TREE phyloFit.NoChrX.placentals.mod | sed 's/TREE\:\ //' \
	> tree_4d.NoChrX.placentals.46way.nh

#########################################################################
# Phylogenetic tree from 46-way  (DONE - 2009-06-25,07-07 - Hiram)
#	This was an early first time experiment.  All this was redone
#	above for chrX only and non-chrX trees

    # Extract 4-fold degenerate sites based on 
    # of RefSeq Reviewed, coding
    mkdir /hive/data/genomes/hg19/bed/multiz46way/4d
    cd /hive/data/genomes/hg19/bed/multiz46way/4d

    hgsql hg19 -Ne \
    "select * from refGene,refSeqStatus where refGene.name=refSeqStatus.mrnaAcc and refSeqStatus.status='Reviewed' and mol='mRNA'" | cut -f 2-20 \
	> refSeqReviewed.gp
    wc -l refSeqReviewed.gp
    # 14077 refSeqReviewed.gp
    genePredSingleCover refSeqReviewed.gp stdout | sort > refSeqReviewedNR.gp
    wc -l refSeqReviewedNR.gp
    # 7951 refSeqReviewedNR.gp

    ssh memk
    mkdir /hive/data/genomes/hg19/bed/multiz46way/4d/run
    cd /hive/data/genomes/hg19/bed/multiz46way/4d/run
    mkdir ../mfa

# whole chrom mafs version, using new version of 
# uses memory-efficient version of phast, from Melissa Hubisz at Cornell
#	mjhubisz at gmail.com
    cat << '_EOF_' > 4d.csh
#!/bin/csh -fe
set r = "/hive/data/genomes/hg19/bed/multiz46way"
set c = $1
set infile = $r/maf/$2
set outfile = $3
cd /scratch/tmp
# 'clean' maf
perl -wpe 's/^s ([^.]+)\.\S+/s $1/' $infile > $c.maf    
awk -v C=$c '$2 == C {print}' $r/4d/refSeqReviewedNR.gp > $c.gp
set PHASTBIN=/cluster/bin/phast.2008-12-18
$PHASTBIN/msa_view --4d --features $c.gp --do-cats 3 -i MAF $c.maf -o SS > $c.ss
$PHASTBIN/msa_view -i SS --tuple-size 1 $c.ss > $r/4d/$outfile
rm -f $c.gp $c.maf $c.ss
'_EOF_'
    # << happy emacs
    chmod +x 4d.csh

    ls -1S /hive/data/genomes/hg19/bed/multiz46way/maf/*.maf | \
        egrep -E -v "chrM|chrUn|random|_hap" | sed -e "s#.*multiz46way/maf/##" \
	> maf.list

    cat << '_EOF_' > template
#LOOP
4d.csh $(root1) {check in line+ $(path1)} {check out line+ mfa/$(root1).mfa}
#ENDLOOP
'_EOF_'
    # << happy emacs

    gensub2 maf.list single template stdout | tac > jobList
XXX - ready to go here - 2009-07-06
    rm -fr /cluster/data/hg19/bed/multiz46way/4d/mfa
    mkdir /cluster/data/hg19/bed/multiz46way/4d/mfa
    para create jobList
    para try
    para check
    para push

    # combine mfa files
    cd ..
    sed -e "s/ /,/g" ../species.list > species.lst
    /cluster/bin/phast/msa_view --aggregate `cat species.lst` mfa/*.mfa | \
        sed s/"> "/">"/ > 4d.all.mfa

    sed -e 's/,macEug1.*//' species.lst > placentals.lst
    #  XXX this didn't work
    /cluster/bin/phast/msa_view --aggregate `cat placentals.lst` mfa/*.mfa | \
        sed s/"> "/">"/ > 4d.placentals.mfa

    # use phyloFit to create tree model (output is phyloFit.mod)
    set PHASTBIN=/cluster/bin/phast.2008-12-18
    time $PHASTBIN/phyloFit --EM --precision MED --msa-format FASTA \
	--subst-mod REV --tree ../tree-commas.nh 4d.all.mfa
    #	real    111m23.119s
    mv phyloFit.mod phyloFit.all.mod
    grep TREE phyloFit.all.mod | sed 's/TREE\:\ //' > tree_4d.46way.nh

    sed -e 's/.*,choHof1,//' species.lst > notPlacentals.list

    $PHASTBIN/tree_doctor \
        --prune=`cat notPlacentals.list` \
                tree_4d.46way.nh > tree_4d.46way.placental.nh

#############################################################################
# phastCons 46-way (DONE - 2009-09-21,2009-11-10 - Hiram)
    #	was unable to split the full chrom MAF files, now working on the
    #	maf files as they were split up during multiz

    # split 46way mafs into 10M chunks and generate sufficient statistics 
    # files for # phastCons
    ssh swarm
    mkdir -p /hive/data/genomes/hg19/bed/multiz46way/cons/msa.split
    cd /hive/data/genomes/hg19/bed/multiz46way/mafSplit
    ./splitRegions.pl mafSplit.bed > \
	/hive/data/genomes/hg19/bed/multiz46way/cons/msa.split/region.list
    mkdir /hive/data/genomes/hg19/bed/multiz46way/cons/ss
    mkdir -p /hive/data/genomes/hg19/bed/multiz46way/cons/msa.split/2009-10-19
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/msa.split/2009-10-19
    mkdir ss

    cat << '_EOF_' > doSplit.csh
#!/bin/csh -ef
set c = $1
set MAF = /hive/data/genomes/hg19/bed/multiz46way/splitRun/maf/hg19_$c.maf
set WINDOWS = /hive/data/genomes/hg19/bed/multiz46way/cons/msa.split/2009-10-19/ss/$c
set WC = `cat $MAF | wc -l`
set NL = `grep "^#" $MAF | wc -l`
if ( -s $2 ) then
    exit 0
endif
if ( -s $2.running ) then
    exit 0
endif

date >> $2.running

rm -fr $WINDOWS
mkdir $WINDOWS
pushd $WINDOWS > /dev/null
if ( $WC != $NL ) then
/cluster/bin/phast.build/cornellCVS/phast.2009-10-19/bin/msa_split \
    $MAF -i MAF -o SS -r $WINDOWS/$c -w 10000000,0 -I 1000 -B 5000
endif
popd > /dev/null
date >> $2
rm -f $2.running
'_EOF_'
    # << happy emacs
    chmod +x doSplit.csh

    cat << '_EOF_' > template
#LOOP
doSplit.csh $(root1) {check out line+ $(root1).done}
#ENDLOOP
'_EOF_'
    # << happy emacs

    #	do the easy ones first to see some immediate results
    ls -1S -r ../../../splitRun/maf | sed -e "s/.maf//; s/hg19_//" > maf.list

    gensub2 maf.list single template jobList
    para -ram=32g create jobList
    para try ... check ... etc
# Completed: 503 of 504 jobs
# Crashed: 1 jobs
# CPU time in finished jobs:      14171s     236.18m     3.94h    0.16d  0.000 y
# IO & Wait Time:                188193s    3136.55m    52.28h    2.18d  0.006 y
# Average job time:                 402s       6.71m     0.11h    0.00d
# Longest finished job:            1597s      26.62m     0.44h    0.02d
# Submission to last job:          2586s      43.10m     0.72h    0.03d
    #	the one crashed job is hg19_chr18_gl000207_random.00.maf

    #	XXX - this did not work
    #	this takes a really long time.  memk was down to 2 usable
    #	machines - got it finished manually on a combination of hgwdevnew CPUs
    #	and other machines

    # Estimate phastCons parameters
    #	experimented with this as a parasol job on hgwdevnew to try a number
    #	of SS files.  With a command of:

/cluster/bin/phast/x86_64/phyloFit -i SS ${SS} \
--tree "(((((((((((((((((hg19,panTro2),gorGor1),ponAbe2),rheMac2),calJac1),tarSyr1),(micMur1,otoGar1)),tupBel1),(((((mm9,rn4),dipOrd1),cavPor3),speTri1),(oryCun1,ochPri2))),(((vicPac1,(turTru1,bosTau4)),((equCab2,(felCat3,canFam2)),(myoLuc1,pteVam1))),(eriEur1,sorAra1))),(((loxAfr2,proCap1),echTel1),(dasNov2,choHof1))),monDom4),ornAna1),((galGal3,taeGut1),anoCar1)),xenTro2),(((tetNig1,fr2),(gasAcu1,oryLat2)),danRer5)),petMar1)" \
--out-root=$OUT/starting_tree

    #	running over the input files ../ss/*/*.ss results to
#.../genomes/hg19/bed/multiz46way/cons/startingTree/result/*/starting-tree.mod

    # add up the C and G:
    find ./result -type f | xargs ls -rt | while read F
do
    D=`dirname $F`
    echo -n `basename $D`" - "
    grep BACKGROUND ${F} | awk '{printf "%0.3f\n", $3 + $4;}'
done
    #	counting number of species seen in the maf file:
    find ./result -type f | xargs ls -rt | while read F
do
    D=`dirname $F`
    echo -n `basename $D`" - "
    grep TREE $F | sed -e \
"s/TREE: //; s/(//g; s/)//g; s/[0-9].[0-9][0-9][0-9][0-9][0-9][0-9]//g; s/://g"  | tr ',' '\n' | wc -l
done

    # Run phastCons
    #	This job is I/O intensive in its output files, beware where this
    #	takes place or do not run too many at once.
    ssh swarm
    mkdir -p /hive/data/genomes/hg19/bed/multiz46way/cons/run.cons
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/run.cons

    #	there are going to be several different phastCons runs using
    #	this same script.  They trigger off of the current working directory
    #	$cwd:t which is the "grp" in this script.  It is one of:
    #	all primates placentals

    cat << '_EOF_' > doPhast.csh
#!/bin/csh -fe
set PHASTBIN = /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin
set c = $1
set cX = $1:r
set f = $2
set len = $3
set cov = $4
set rho = $5
set grp = $cwd:t
set cons = /hive/data/genomes/hg19/bed/multiz46way/cons
set tmp = $cons/tmp/$f
mkdir -p $tmp
set ssSrc = $cons
set useGrp = "$grp.mod"
if ( $cX == "chrX" ) then
    set useGrp = "$grp.chrX.mod"
endif
if (-s $cons/$grp/$grp.non-inf) then
  ln -s $cons/$grp/$grp.mod $tmp
  ln -s $cons/$grp/$grp.chrX.mod $tmp
  ln -s $cons/$grp/$grp.non-inf $tmp
  ln -s $ssSrc/msa.split/2009-10-21/ss/$c/$f.ss $tmp
else
  ln -s $ssSrc/msa.split/2009-10-21/ss/$c/$f.ss $tmp
  ln -s $cons/$grp/$grp.mod $tmp
  ln -s $cons/$grp/$grp.chrX.mod $tmp
endif
pushd $tmp > /dev/null
if (-s $grp.non-inf) then
  $PHASTBIN/phastCons $f.ss $useGrp \
    --rho $rho --expected-length $len --target-coverage $cov --quiet \
    --not-informative `cat $grp.non-inf` \
    --seqname $c --idpref $c --most-conserved $f.bed --score > $f.pp
else
  $PHASTBIN/phastCons $f.ss $useGrp \
    --rho $rho --expected-length $len --target-coverage $cov --quiet \
    --seqname $c --idpref $c --most-conserved $f.bed --score > $f.pp
endif
popd > /dev/null
mkdir -p pp/$c bed/$c
sleep 4
touch pp/$c bed/$c
rm -f pp/$c/$f.pp
rm -f bed/$c/$f.bed
mv $tmp/$f.pp pp/$c
mv $tmp/$f.bed bed/$c
rm -fr $tmp
'_EOF_'
    # << happy emacs
    chmod a+x doPhast.csh

    #	this template will serve for all runs
    #	root1 == chrom name, file1 == ss file name without .ss suffix
    cat << '_EOF_' > template
#LOOP
../run.cons/doPhast.csh $(root1) $(file1) 45 0.3 0.3 {check out line+ pp/$(root1)/$(file1).pp}
#ENDLOOP
'_EOF_'
    # << happy emacs

    ls -1S ../msa.split/2009-10-21/ss/chr*/chr* | sed -e "s/.ss$//" > ss.list

    # Create parasol batch and run it
    # run for all species
    cd /hive/data/genomes/hg19/bed/multiz46way/cons
    mkdir -p all
    cd all
    #	Using the two different .mod tree
    cp -p ../../4dNoX/phyloFit.NoChrX.mod ./all.mod
    cp -p ../../4dX/phyloFit.chrX.mod ./all.chrX.mod

    gensub2 ../run.cons/ss.list single ../run.cons/template jobList
    para -ram=8g create jobList
    para try ... check ... push ... etc.
# Completed: 581 of 581 jobs
# CPU time in finished jobs:      41877s     697.95m    11.63h    0.48d  0.001 y
# IO & Wait Time:                 39172s     652.87m    10.88h    0.45d  0.001 y
# Average job time:                 139s       2.32m     0.04h    0.00d
# Longest finished job:             329s       5.48m     0.09h    0.00d
# Submission to last job:          2240s      37.33m     0.62h    0.03d

    # create Most Conserved track
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/all
    cut -f1 ../../../../chrom.sizes | while read C
do  
    ls -d bed/${C}.[0-9][0-9] 2> /dev/null | while read D
    do
        cat ${D}/${C}*.bed
    done | sort -k1,1 -k2,2n \
    | awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", "'${C}'", $2, $3, $5, $5;}'
done > tmpMostConserved.bed
/cluster/bin/scripts/lodToBedScore tmpMostConserved.bed > mostConserved.bed

    # load into database
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/all
    time nice -n +19 hgLoadBed hg19 phastConsElements46way mostConserved.bed
    #	Loaded 5163775 elements of size 6
    #	real     1m44.439s

    # Try for 5% overall cov, and 70% CDS cov 
    featureBits hg19 -enrichment refGene:cds phastConsElements46way
    #	--rho 0.3 --expected-length 45 --target-coverage 0.3
    #	refGene:cds 1.187%, phastConsElements46way 5.065%,
    #	both 0.884%, cover 74.46%, enrich 14.70x

    # Create merged posterier probability file and wiggle track data files
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/all
    mkdir downloads
    cat << '_EOF_' > phastCat.sh
#!/bin/sh

mkdir -p downloads
cut -f1 ../../../../chrom.sizes | while read C
do
    echo -n "${C} ... working ... "
    ls -d pp/${C}.[0-9][0-9] 2> /dev/null | while read D
    do
        cat ${D}/${C}*.pp | sed -e "s/chrom=${C}.[0-9][0-9]/chrom=${C}/"
    done | gzip > downloads/${C}.phastCons46way.wigFix.gz
    echo "done"
done
'_EOF_'
    #	<< happy emacs
    chmod +x phastCat.sh
    time nice -n +19 ./phastCat.sh
    #	real    30m2.623s

    #	encode those files into wiggle data
    zcat downloads/*.wigFix.gz \
	| wigEncode stdin phastCons46way.wig phastCons46way.wib
    #	Converted stdin, upper limit 1.00, lower limit 0.00
    #	real    18m37.881s
    du -hsc *.wi?
    #	2.7G    phastCons46way.wib
    #	271M    phastCons46way.wig
    #	3.0G    total

    #	encode into a bigWig file:
    #	(warning wigToBigWig process grows to about 36 Gb)
    #	in bash, to avoid the 32 Gb memory limit:
sizeG=188743680
export sizeG
ulimit -d $sizeG 
ulimit -v $sizeG
    zcat downloads/*.wigFix.gz \
        | wigToBigWig stdin ../../../../chrom.sizes phastCons46way.bw
    #	real    52m36.142s
# -rw-rw-r--   1 21667535139 Oct 20 13:59 phastCons46way.bw
    mkdir /gbdb/hg19/bbi
    ln -s `pwd`/phastCons46way.bw /gbdb/hg19/bbi
    #	if you wanted to use the bigWig file, loading bigWig table:
    hgsql hg19 -e 'drop table if exists phastCons46way; \
            create table phastCons46way (fileName varchar(255) not null); \
            insert into phastCons46way values
	("/gbdb/hg19/bbi/phastCons46way.bw");'

    # Load gbdb and database with wiggle.
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/all
    ln -s `pwd`/phastCons46way.wib /gbdb/hg19/multiz46way/phastCons46way.wib
    time nice -n +19 hgLoadWiggle -pathPrefix=/gbdb/hg19/multiz46way hg19 \
	phastCons46way phastCons46way.wig
    #	real    1m45.381s

    wigTableStats.sh hg19 phastCons46way
# db.table      min max mean count sumData
# hg19.phastCons46way     0 1 0.103653 2845303719 2.94924e+08
#	stdDev viewLimits
#	0.230184 viewLimits=0:1

    #  Create histogram to get an overview of all the data
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/all
    time nice -n +19 hgWiggle -doHistogram -db=hg19 \
	-hBinSize=0.001 -hBinCount=1000 -hMinVal=0.0 -verbose=2 \
	    phastCons46way > histogram.data 2>&1
    #	real    7m37.212s

    #	create plot of histogram:

    cat << '_EOF_' | gnuplot > histo.png
set terminal png small color x000000 xffffff xc000ff x66ff66 xffff00 x00ffff
set size 1.4, 0.8
set key left box
set grid noxtics
set grid ytics
set title " Human Hg19 Histogram phastCons46way track"
set xlabel " phastCons46way score"
set ylabel " Relative Frequency"
set y2label " Cumulative Relative Frequency (CRF)"
set y2range [0:1]
set y2tics
set yrange [0:0.02]

plot "histogram.data" using 2:5 title " RelFreq" with impulses, \
        "histogram.data" using 2:7 axes x1y2 title " CRF" with lines
'_EOF_'
    #	<< happy emacs

    display histo.png &

    ########################################################################
    ### Create a phastCons data set for Primates

    # setup primates-only run
    ssh swarm
    mkdir /hive/data/genomes/hg19/bed/multiz46way/cons/primates
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/primates
    # primates-only: exclude all but these for phastCons tree:

    cp -p ../../4dNoX/phyloFit.NoChrX.primates.mod primates.mod
    cp -p ../../4dX/phyloFit.chrX.primates.mod primates.chrX.mod
    #	and place the removed ones in the non-inf file so phastCons will
    #	truly ignore them:
    echo "tupBel1,mm9,rn4,dipOrd1,cavPor3,speTri1,oryCun2,ochPri2,vicPac1,turTru1,bosTau4,equCab2,felCat3,canFam2,myoLuc1,pteVam1,eriEur1,sorAra1,loxAfr3,proCap1,echTel1,dasNov2,choHof1,macEug1,monDom5,ornAna1,galGal3,taeGut1,anoCar1,xenTro2,tetNig2,fr2,gasAcu1,oryLat2,danRer6,petMar1" \
	> primates.non-inf

    gensub2 ../run.cons/ss.list single ../run.cons/template jobList
    para -ram=8g create jobList
    para try ... check ... push ... etc.
# Completed: 581 of 581 jobs
# CPU time in finished jobs:      17077s     284.62m     4.74h    0.20d  0.001 y
# IO & Wait Time:                 73693s    1228.21m    20.47h    0.85d  0.002 y
# Average job time:                 156s       2.60m     0.04h    0.00d
# Longest finished job:             402s       6.70m     0.11h    0.00d
# Submission to last job:          2322s      38.70m     0.65h    0.03d

    cd /hive/data/genomes/hg19/bed/multiz46way/cons/primates
    # create Most Conserved track
    cut -f1 ../../../../chrom.sizes | while read C
do  
    ls -d bed/${C}.[0-9][0-9] 2> /dev/null | while read D
    do
        cat ${D}/${C}*.bed
    done | sort -k1,1 -k2,2n \
    | awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", "'${C}'", $2, $3, $5, $5;}'
done > tmpMostConserved.bed
/cluster/bin/scripts/lodToBedScore tmpMostConserved.bed > mostConserved.bed

    featureBits hg19 mostConserved.bed
    #	146285948 bases of 2897316137 (5.049%) in intersection

    # load into database
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/primates
    time nice -n +19 hgLoadBed hg19 phastConsElements46wayPrimates \
	mostConserved.bed
    #	Loaded 725627 elements of size 6
    #	real    0m8.583s
    # verify coverage
    featureBits hg19 phastConsElements46wayPrimates
    #	116785954 bases of 2897316137 (4.031%) in intersection

    #	--rho 0.3 --expected-length 45 --target-coverage 0.3
    featureBits hg19 -enrichment refGene:cds phastConsElements46wayPrimates
    #	refGene:cds 1.186%, phastConsElements46wayPrimates 4.031%,
    #	both 0.730%, cover 61.54%, enrich 15.27x

    featureBits hg19 -enrichment knownGene:cds phastConsElements46wayPrimates
    #	knownGene:cds 1.252%, phastConsElements46wayPrimates 4.031%,
    #	both 0.743%, cover 59.31%, enrich 14.71x

    #	Create the downloads .pp files, from which the phastCons wiggle data
    #	is calculated
    # sort by chromName, chromStart so that items are in numerical order 
    #  for wigEncode
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/primates
    mkdir downloads
    cat << '_EOF_' > phastCat.sh
#!/bin/sh

mkdir -p downloads
cut -f1 ../../../../chrom.sizes | while read C
do
    echo -n "${C} ... working ... "
    ls -d pp/${C}.[0-9][0-9] 2> /dev/null | while read D
    do
	cat ${D}/${C}*.pp | sed -e "s/chrom=${C}.[0-9][0-9]/chrom=${C}/"
    done | gzip > downloads/${C}.phastCons46way.primates.wigFix.gz
    echo "done"
done
'_EOF_'
    # << happy emacs
    chmod +x ./phastCat.sh
    time nice -n +19 ./phastCat.sh
    #	real    39m47.189s

    # Create merged posterier probability file and wiggle track data files
    zcat downloads/chr*.wigFix.gz \
	 | wigEncode stdin phastCons46wayPrimates.wig phastCons46wayPrimates.wib
    # Converted stdin, upper limit 1.00, lower limit 0.00
    #	real    17m20.601s

    #	encode to bigWig
    #	(warning wigToBigWig process grows to about 36 Gb)
    #	in bash, to avoid the 32 Gb memory limit:
sizeG=188743680
export sizeG
ulimit -d $sizeG 
ulimit -v $sizeG

    zcat downloads/*.wigFix.gz \
        | wigToBigWig stdin ../../../../chrom.sizes phastCons46wayPrimates.bw

    ln -s `pwd`/phastCons46wayPrimates.bw /gbdb/hg19/bbi
    #	if desired to use the bigWig file, loading bigWig table:
    hgsql hg19 -e 'drop table if exists phastCons46wayPrimates; \
            create table phastCons46wayPrimates \
		(fileName varchar(255) not null); \
            insert into phastCons46wayPrimates values
	("/gbdb/hg19/bbi/phastCons46wayPrimates.bw");'

    ## load table with wiggle data
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/primates
    ln -s `pwd`/phastCons46wayPrimates.wib \
	/gbdb/hg19/multiz46way/phastCons46wayPrimates.wib
    time nice -n +19 hgLoadWiggle -pathPrefix=/gbdb/hg19/multiz46way hg19 \
	phastCons46wayPrimates phastCons46wayPrimates.wig

    wigTableStats.sh hg19 phastCons46wayPrimates
# db.table      min max mean count sumData
hg19.phastCons46wayPrimates     0 1 0.128883 2845303719 3.66712e+08
#	stdDev viewLimits
#	0.214067 viewLimits=0:1

    #  Create histogram to get an overview of all the data
    time nice -n +19 hgWiggle -doHistogram \
	-hBinSize=0.001 -hBinCount=1000 -hMinVal=0.0 -verbose=2 \
	    -db=hg19 phastCons46wayPrimates  > histogram.data 2>&1
    #	real    5m30.086s

    #	create plot of histogram:

    cat << '_EOF_' | gnuplot > histo.png
set terminal png small color \
        x000000 xffffff xc000ff x66ff66 xffff00 x00ffff xff0000
set size 1.4, 0.8
set key left box
set grid noxtics
set grid ytics
set title " Mouse Hg19 Histogram phastCons46wayPrimates track"
set xlabel " phastCons46wayPrimates score"
set ylabel " Relative Frequency"
set y2label " Cumulative Relative Frequency (CRF)"
set y2range [0:1]
set y2tics
set yrange [0:0.02]

plot "histogram.data" using 2:5 title " RelFreq" with impulses, \
        "histogram.data" using 2:7 axes x1y2 title " CRF" with lines
'_EOF_'
    #	<< happy emacs

    display histo.png &

    ########################################################################
    ### Create a phastCons data set for Placentals
    # setup placental-only run
    ssh swarm
    mkdir /hive/data/genomes/hg19/bed/multiz46way/cons/placental
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/placental

    cp -p ../../4dNoX/phyloFit.NoChrX.placentals.mod placental.mod
    cp -p ../../4dX/phyloFit.chrX.placentals.mod placental.chrX.mod
    # placental-only: exclude all but these for phastCons tree:
    #	and place the removed ones in the non-inf file so phastCons will
    #	truly ignore them:
    echo "macEug1,monDom5,ornAna1,galGal3,taeGut1,anoCar1,xenTro2,tetNig2,fr2,gasAcu1,oryLat2,danRer6,petMar1" \
        > placental.non-inf

    gensub2 ../run.cons/ss.list single ../run.cons/template jobList
    para -ram=8g create jobList
    para try ... check ... push ... etc.
# Completed: 581 of 581 jobs
# CPU time in finished jobs:      33942s     565.69m     9.43h    0.39d  0.001 y
# IO & Wait Time:                 75536s    1258.94m    20.98h    0.87d  0.002 y
# Average job time:                 188s       3.14m     0.05h    0.00d
# Longest finished job:             417s       6.95m     0.12h    0.00d
# Submission to last job:          1878s      31.30m     0.52h    0.02d

    # create Most Conserved track
    cut -f1 ../../../../chrom.sizes | while read C
do  
    ls -d bed/${C}.[0-9][0-9] 2> /dev/null | while read D
    do
        cat ${D}/${C}*.bed
    done | sort -k1,1 -k2,2n \
    | awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", "'${C}'", $2, $3, $5, $5;}'
done > tmpMostConserved.bed
/cluster/bin/scripts/lodToBedScore tmpMostConserved.bed > mostConserved.bed

    # load into database
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/placental
    time nice -n +19 hgLoadBed hg19 phastConsElements46wayPlacental \
	mostConserved.bed
    #	Loaded 3743478 elements of size 6
    #	real    1m15.952s
    # verify coverage
    featureBits hg19 phastConsElements46wayPlacental
    #	118211444 bases of 2897316137 (4.080%) in intersection

    #	--rho 0.3 --expected-length 45 --target-coverage 0.3
    featureBits hg19 -enrichment refGene:cds phastConsElements46wayPlacental
    #	refGene:cds 1.187%, phastConsElements46wayPlacental 4.080%,
    #	both 0.861%, cover 72.59%, enrich 17.79x
    featureBits hg19 -enrichment knownGene:cds phastConsElements46wayPlacental
    #	knownGene:cds 1.252%, phastConsElements46wayPlacental 4.080%,
    #	both 0.879%, cover 70.22%, enrich 17.21x

    #	Create the downloads .pp files, from which the phastCons wiggle data
    #	is calculated
    # sort by chromName, chromStart so that items are in numerical order 
    #  for wigEncode
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/placental
    mkdir downloads
    cat << '_EOF_' > phastCat.sh
#!/bin/sh

mkdir -p downloads
cut -f1 ../../../../chrom.sizes | while read C
do
    echo -n "${C} ... working ... "
    ls -d pp/${C}.[0-9][0-9] 2> /dev/null | while read D
    do
	cat ${D}/${C}*.pp | sed -e "s/chrom=${C}.[0-9][0-9]/chrom=${C}/"
    done | gzip > downloads/${C}.phastCons46way.placental.wigFix.gz
    echo "done"
done
'_EOF_'
    # << happy emacs
    chmod +x ./phastCat.sh
    time nice -n +19 ./phastCat.sh

    # Create merged posterier probability file and wiggle track data files
    zcat downloads/chr*.wigFix.gz \
	| wigEncode stdin phastCons46wayPlacental.wig \
		phastCons46wayPlacental.wib
    #	Converted stdin, upper limit 1.00, lower limit 0.00
    #	real    14m53.395s

    #	encode to bigWig
    #	(warning wigToBigWig process grows to about 36 Gb)
    #	in bash, to avoid the 32 Gb memory limit:
sizeG=188743680
export sizeG
ulimit -d $sizeG 
ulimit -v $sizeG

    zcat downloads/*.wigFix.gz \
        | wigToBigWig stdin ../../../../chrom.sizes phastCons46wayPlacental.bw
    #	real    40m55.568s

    ln -s `pwd`/phastCons46wayPlacental.bw /gbdb/hg19/bbi
    #	loading bigWig table:
    hgsql hg19 -e 'drop table if exists phastCons46wayPlacental; \
            create table phastCons46wayPlacental \
		(fileName varchar(255) not null); \
            insert into phastCons46wayPlacental values
	("/gbdb/hg19/bbi/phastCons46wayPlacental.bw");'

    ## load table with wiggle data
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/multiz46way/cons/placental
    ln -s `pwd`/phastCons46wayPlacental.wib \
	/gbdb/hg19/multiz46way/phastCons46wayPlacental.wib
    time nice -n +19 hgLoadWiggle -pathPrefix=/gbdb/hg19/multiz46way hg19 \
	phastCons46wayPlacental phastCons46wayPlacental.wig

    wigTableStats.sh hg19 phastCons46wayPlacental
# db.table      min max mean count sumData
hg19.phastCons46wayPlacental    0 1 0.0885757 2845303719 2.52025e+08
#	stdDev viewLimits
#	0.210242 viewLimits=0:1

    #  Create histogram to get an overview of all the data
    time nice -n +19 hgWiggle -doHistogram \
	-hBinSize=0.001 -hBinCount=1000 -hMinVal=0.0 -verbose=2 \
	    -db=hg19 phastCons46wayPlacental > histogram.data 2>&1
    #	real    8m15.623s

    #	create plot of histogram:

    cat << '_EOF_' | gnuplot > histo.png
set terminal png small color x000000 xffffff xc000ff x66ff66 xffff00 x00ffff
set size 1.4, 0.8
set key left box
set grid noxtics
set grid ytics
set title " Human Hg19 Histogram phastCons46wayPlacental track"
set xlabel " phastCons46wayPlacental score"
set ylabel " Relative Frequency"
set y2label " Cumulative Relative Frequency (CRF)"
set y2range [0:1]
set y2tics
set yrange [0:0.02]

plot "histogram.data" using 2:5 title " RelFreq" with impulses, \
        "histogram.data" using 2:7 axes x1y2 title " CRF" with lines
'_EOF_'
    #	<< happy emacs

    display histo.png &

#########################################################################
# phyloP conservation for 46-way (DONE - 2009-10-21,2009-11-10 - Hiram)
#
# Vertebrate, Placental, Primates
#
    # split SS files into 1M chunks, this business needs smaller files
    #   to complete

    ssh swarm
    mkdir /hive/data/genomes/hg19/bed/multiz46way/consPhyloP
    cd /hive/data/genomes/hg19/bed/multiz46way/consPhyloP
    mkdir ss run.split
    cd run.split

    cat << '_EOF_' > doSplit.csh
#!/bin/csh -ef
set c = $1
set MAF = /hive/data/genomes/hg19/bed/multiz46way/splitRun/maf/hg19_$c.maf
set WINDOWS = /hive/data/genomes/hg19/bed/multiz46way/consPhyloP/run.split/ss/$c
set WC = `cat $MAF | wc -l`
set NL = `grep "^#" $MAF | wc -l`
if ( -s $2 ) then
    exit 0
endif
if ( -s $2.running ) then
    exit 0
endif

date >> $2.running

rm -fr $WINDOWS
mkdir $WINDOWS
pushd $WINDOWS > /dev/null
if ( $WC != $NL ) then
/cluster/bin/phast.build/cornellCVS/phast.2009-10-19/bin/msa_split \
    $MAF -i MAF -o SS -r $WINDOWS/$c -w 1000000,0 -I 1000 -B 5000
endif
popd > /dev/null
date >> $2
rm -f $2.running
'_EOF_'
# << happy emacs

    ls -1S -r ../../splitRun/maf | sed -e "s/.maf//; s/hg19_//" > maf.list

    cat << '_EOF_' > template
#LOOP
doSplit.csh $(path1) {check out exists+ done/$(path1).done}
#ENDLOOP
'_EOF_'
# << happy emacs

    mkdir ss done
    gensub2 maf.list single template jobList
    para -ram=8g create jobList
# Completed: 504 of 504 jobs
# CPU time in finished jobs:      14486s     241.43m     4.02h    0.17d  0.000 y
# IO & Wait Time:                306280s    5104.67m    85.08h    3.54d  0.010 y
# Average job time:                 636s      10.61m     0.18h    0.01d
# Longest finished job:            1635s      27.25m     0.45h    0.02d
# Submission to last job:          2965s      49.42m     0.82h    0.03d


    # run phyloP with score=LRT 
    ssh swarm
    cd /cluster/data/hg18/bed/multiz44way/consPhyloP
    mkdir run.phyloP
    cd run.phyloP

    # Adjust model file base composition background and rate matrix to be
    # representative of the chromosomes in play
    grep BACKGROUND ../../cons/all/all.mod | awk '{printf "%0.3f\n", $3 + $4}'
    #	0.542
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/modFreqs \
	../../cons/all/all.mod 0.542 > all.mod
    grep BACKGROUND ../../cons/all/all.chrX.mod \
	| awk '{printf "%0.3f\n", $3 + $4}'
    #	0.503
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/modFreqs \
	../../cons/all/all.chrX.mod 0.503 > all.chrX.mod
    grep BACKGROUND ../../cons/primates/primates.mod \
	| awk '{printf "%0.3f\n", $3 + $4}'
    #	0.523
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/modFreqs \
	../../cons/primates/primates.mod 0.523 > primates.mod
    grep BACKGROUND ../../cons/primates/primates.chrX.mod \
	| awk '{printf "%0.3f\n", $3 + $4}'
    #	0.491
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/modFreqs \
	../../cons/primates/primates.chrX.mod 0.491 > primates.chrX.mod
    grep BACKGROUND ../../cons/placental/placental.mod \
	| awk '{printf "%0.3f\n", $3 + $4}'
    #	0.542
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/modFreqs \
	../../cons/placental/placental.mod 0.542 > placental.mod
    grep BACKGROUND ../../cons/placental/placental.chrX.mod \
	| awk '{printf "%0.3f\n", $3 + $4}'
    #	0.489
    /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin/modFreqs \
	../../cons/placental/placental.chrX.mod 0.489 > placental.chrX.mod


    # repeat for chrX only tree
    cd /cluster/data/hg18/bed/multiz46way/4d
    $PHASTBIN/modFreqs 4d.chrX.mod $gc > 46way.chrX.mod
    ln -s `pwd`/46way.chrX.mod /usr/local/apache/golenPath/hg18/phastCons46way

    cat << '_EOF_' > doPhyloP.csh
#!/bin/csh -fe
set PHASTBIN = /cluster/bin/phast.build/cornellCVS/phast.2009-10-21/bin
set f = $1
set out = $2
set cName = $f:r:r
set chrDir = $f:r
set n = $f:r:e
set grp = $cwd:t
set cons = /hive/data/genomes/hg19/bed/multiz46way/consPhyloP
set tmp = $cons/tmp/$grp/$f
rm -fr $tmp
mkdir -p $tmp
set ssSrc = "$cons/run.split/ss/$chrDir/$f"
set useGrp = "$grp.mod"
if ( $cName == "chrX" ) then
    set useGrp = "$grp.chrX.mod"
endif
ln -s $cons/run.phyloP/$grp.mod $tmp
ln -s $cons/run.phyloP/$grp.chrX.mod $tmp
pushd $tmp > /dev/null
$PHASTBIN/phyloP --method LRT --mode CONACC --wig-scores --chrom $cName \
    -i SS $useGrp $ssSrc.ss > $f.wigFix
popd > /dev/null
mkdir -p $out:h
sleep 4
mv $tmp/$f.wigFix $out
rm -fr $tmp
'_EOF_'
    # << happy emacs

    # Create list of chunks
    find ../run.split/ss -type f | sed -e "s/.ss$//; s#^../run.split/ss/##" \
	> ss.list

    # Create template file
    #	file1 == $chr/$chunk/file name without .ss suffix
    cat << '_EOF_' > template
#LOOP
../run.phyloP/doPhyloP.csh $(file1) {check out line+ wigFix/$(dir1)/$(file1).wigFix}
#ENDLOOP
'_EOF_'
    # << happy emacs

    ######################   Running all species  #######################
    # setup run for all species
    mkdir /hive/data/genomes/hg19/bed/multiz46way/consPhyloP/all
    cd /hive/data/genomes/hg19/bed/multiz46way/consPhyloP/all
    rm -fr wigFix
    mkdir wigFix

    gensub2 ../run.phyloP/ss.list single ../run.phyloP/template jobList
    para create jobList
    para try ... check ... push ... etc ...

    para time
# Completed: 3186 of 3186 jobs
# CPU time in finished jobs:    1306874s   21781.23m   363.02h   15.13d  0.041 y
# IO & Wait Time:                105488s    1758.14m    29.30h    1.22d  0.003 y
# Average job time:                 443s       7.39m     0.12h    0.01d
# Longest finished job:             678s      11.30m     0.19h    0.01d
# Submission to last job:          7789s     129.82m     2.16h    0.09d

    ssh hgwdev
    cd /cluster/data/hg18/bed/multiz46way/consPhyloP/run.phyloP/all
    find ./wigFix -type f \
	| sed -e "s#^./##; s/\./ /g; s/-/ - /g" \
	| sort -k1,1 -k3,3n -k4,4n | sed -e "s/ - /-/g; s/ /./g" > wigFile.list
    cat wigFile.list | xargs cat \
	| wigEncode stdin phyloP46way.wig phyloP46way.wib > wigEncode.log 2>&1 &
    #	Converted stdin, upper limit 6.39, lower limit -13.27
    cat wigFile.list | xargs cat \
	| wigToBigWig stdin ../../../../chrom.sizes phyloP46way.bw
    #	if you wanted to use the bigWig file, loading bigWig table:
    ln -s `pwd`/phyloP46way.bw /gbdb/hg19/bbi
    hgsql hg19 -e 'drop table if exists phyloP46wayAll; \
            create table phyloP46wayAll \
		(fileName varchar(255) not null); \
            insert into phyloP46wayAll values
	("/gbdb/hg19/bbi/phyloP46way.bw");'

    #	loading the wiggle table:
    ln -s `pwd`/phyloP46way.wib /gbdb/hg19/multiz46way
    time nice -n +19 hgLoadWiggle -pathPrefix=/gbdb/hg19/multiz46way hg19 \
	phyloP46wayAll phyloP46way.wig

    #	create download files:
    cat << '_EOF_' > mkDown.csh
#!/bin/csh -fe
foreach F (`cat wigFile.list`)
    set C = $F:h:t:r
    cat $F >> downloads/${C}.wigFix
end
'_EOF_'
    # << happy emacs
    chmod +x ./mkDown.csh
    mkdir downloads
    time ./mkDown.csh
    #	real    16m19.683s

    time gzip downloads/chr*.wigFix
    #	real    47m11.017s

    wigTableStats.sh hg19 phyloP46wayAll
# db.table      min max mean count sumData
# hg19.phyloP46wayAll     -14.08 6.424 0.0896064 2845303719 2.54957e+08
#	stdDev viewLimits
#	0.833186 viewLimits=-4.07632:4.25553
    #	that range is: 14.08+6.424 = 20.504

    #  Create histogram to get an overview of all the data
    time nice -n +19 hgWiggle -doHistogram \
	-hBinSize=0.020504 -hBinCount=1000 -hMinVal=-14.08 -verbose=2 \
	    -db=hg19 phyloP46wayAll > histogram.data 2>&1
    #	real    8m15.623s

    #	create plot of histogram:

    cat << '_EOF_' | gnuplot > histo.png
set terminal png small color x000000 xffffff xc000ff x66ff66 xffff00 x00ffff
set size 1.4, 0.8
set key left box
set grid noxtics
set grid ytics
set title " Human Hg19 Histogram phyloP46way track, all 46 vertebrates"
set xlabel " phyloP46way score, all 46 vertebrates"
set ylabel " Relative Frequency"
set y2label " Cumulative Relative Frequency (CRF)"
set y2range [0:1]
set y2tics
set yrange [0:0.04]
set xrange [-2:2]

plot "histogram.data" using 2:5 title " RelFreq" with impulses, \
        "histogram.data" using 2:7 axes x1y2 title " CRF" with lines
'_EOF_'
    #	<< happy emacs

    display histo.png &

    ######################   Running the primates  #######################
    mkdir /hive/data/genomes/hg19/bed/multiz46way/consPhyloP/primates
    cd /hive/data/genomes/hg19/bed/multiz46way/consPhyloP/primates
    rm -fr wigFix
    mkdir wigFix

    gensub2 ../run.phyloP/ss.list single ../run.phyloP/template jobList
    para create jobList
    para try ... check ... push ... etc ...

    para time
# Completed: 3186 of 3186 jobs
# CPU time in finished jobs:     447177s    7452.95m   124.22h    5.18d  0.014 y
# IO & Wait Time:                 36673s     611.22m    10.19h    0.42d  0.001 y
# Average job time:                 152s       2.53m     0.04h    0.00d
# Longest finished job:             279s       4.65m     0.08h    0.00d
# Submission to last job:          4849s      80.82m     1.35h    0.06d

    cd /cluster/data/hg18/bed/multiz46way/consPhyloP/run.phyloP/primates
    find ./wigFix -type f \
	| sed -e "s#^./##; s/\./ /g; s/-/ - /g" \
	| sort -k1,1 -k3,3n -k4,4n | sed -e "s/ - /-/g; s/ /./g" > wigFile.list
    cat wigFile.list | xargs cat \
	| wigEncode stdin phyloP46wayPrimates.wig phyloP46wayPrimates.wib \
	> wigEncode.log 2>&1 &
    #	Converted stdin, upper limit 0.65, lower limit -9.12
    cat wigFile.list | xargs cat \
	| wigToBigWig stdin ../../../../chrom.sizes phyloP46wayPrimates.bw
    #	if you wanted to use the bigWig file, loading bigWig table:
    ln -s `pwd`/phyloP46wayPrimates.bw /gbdb/hg19/bbi
    hgsql hg19 -e 'drop table if exists phyloP46wayPrimates; \
            create table phyloP46wayPrimates \
		(fileName varchar(255) not null); \
            insert into phyloP46wayPrimates values
	("/gbdb/hg19/bbi/phyloP46wayPrimates.bw");'

    #	loading the wiggle table:
    ln -s `pwd`/phyloP46wayPrimates.wib /gbdb/hg19/multiz46way
    time nice -n +19 hgLoadWiggle -pathPrefix=/gbdb/hg19/multiz46way hg19 \
	phyloP46wayPrimates phyloP46wayPrimates.wig

    #	create download files:
    mkdir downloads
    time ../all/mkDown.csh
    #	real    18m44.186s
    time gzip downloads/chr*.wigFix
    #	real    32m11.461s

    wigTableStats.sh hg19 phyloP46wayPrimates
# db.table      min max mean count
# hg19.phyloP46wayPrimates        -9.065 0.655 0.0448196 2845303719
#	sumData stdDev viewLimits
#	1.27525e+08 0.600051 viewLimits=-2.95544:0.655
    #	that range is: 9.065+0.655 = 9.720

    #  Create histogram to get an overview of all the data
    time nice -n +19 hgWiggle -doHistogram \
	-hBinSize=0.00972 -hBinCount=1000 -hMinVal=-9.065 -verbose=2 \
	    -db=hg19 phyloP46wayPrimates > histogram.data 2>&1
    #	real    8m15.623s

    #	create plot of histogram:

    cat << '_EOF_' | gnuplot > histo.png
set terminal png small color x000000 xffffff xc000ff x66ff66 xffff00 x00ffff
set size 1.4, 0.8
set key left box
set grid noxtics
set grid ytics
set title " Human Hg19 Histogram phyloP46wayPrimates track"
set xlabel " phyloP46wayPrimates score"
set ylabel " Relative Frequency"
set y2label " Cumulative Relative Frequency (CRF)"
set y2range [0:1]
set y2tics
set yrange [0:0.03]
set xrange [-2:0.655]

plot "histogram.data" using 2:5 title " RelFreq" with impulses, \
        "histogram.data" using 2:7 axes x1y2 title " CRF" with lines
'_EOF_'
    #	<< happy emacs

    display histo.png &

    ######################   Running the placentals  #######################
    mkdir /hive/data/genomes/hg19/bed/multiz46way/consPhyloP/placentals
    cd /hive/data/genomes/hg19/bed/multiz46way/consPhyloP/placentals
    rm -fr wigFix
    mkdir wigFix

    gensub2 ../run.phyloP/ss.list single ../run.phyloP/template jobList
    para create jobList
    para try ... check ... push ... etc ...
    para time
# Completed: 3186 of 3186 jobs
# CPU time in finished jobs:    1582989s   26383.14m   439.72h   18.32d  0.050 y
rY.phyloP46way.placental.wigFix.gz
# IO & Wait Time:                 25577s     426.29m     7.10h    0.30d  0.001 y
# Average job time:                 505s       8.41m     0.14h    0.01d
# Longest finished job:             768s      12.80m     0.21h    0.01d
# Submission to last job:         12967s     216.12m     3.60h    0.15d

    cd /cluster/data/hg18/bed/multiz46way/consPhyloP/run.phyloP/placental
    find ./wigFix -type f \
	| sed -e "s#^./##; s/\./ /g; s/-/ - /g" \
	| sort -k1,1 -k3,3n -k4,4n | sed -e "s/ - /-/g; s/ /./g" > wigFile.list
    cat wigFile.list | xargs cat \
	| wigEncode stdin phyloP46wayPlacental.wig phyloP46wayPlacental.wib \
	> wigEncode.log 2>&1 &
    #	Converted stdin, upper limit 2.95, lower limit -13.28
    cat wigFile.list | xargs cat \
	| wigToBigWig stdin ../../../../chrom.sizes phyloP46wayPlacental.bw

    #	loading bigWig table:
    ln -s `pwd`/phyloP46wayPlacental.bw /gbdb/hg19/bbi
    hgsql hg19 -e 'drop table if exists phyloP46wayPlacental; \
            create table phyloP46wayPlacental \
		(fileName varchar(255) not null); \
            insert into phyloP46wayPlacental values
	("/gbdb/hg19/bbi/phyloP46wayPlacental.bw");'

    #	loading the wiggle table:
    ln -s `pwd`/phyloP46wayPlacental.wib /gbdb/hg19/multiz46way
    time hgLoadWiggle
    time nice -n +19 hgLoadWiggle -pathPrefix=/gbdb/hg19/multiz46way hg19 \
	phyloP46wayPlacental phyloP46wayPlacental.wig

    #	create download files:
    mkdir downloads
    time ../all/mkDown.csh
    #	real    18m52.778s
    time gzip downloads/chr*.wigFix
    #	real    46m55.550s

    wigTableStatus.sh hg19 phyloP46wayPlacental
# db.table      min max mean count sumData stdDev viewLimits
# hg19.phyloP46wayPlacental -13.796 2.941 0.0359345 2845303719 1.02245e+08
#	stdDev viewLimits
#	0.779426 viewLimits=-3.86119:2.941
    #	that range is: 13.796+2.941 = 16.737

    #  Create histogram to get an overview of all the data
    time nice -n +19 hgWiggle -doHistogram \
	-hBinSize=0.016737 -hBinCount=1000 -hMinVal=-13.796 -verbose=2 \
	    -db=hg19 phyloP46wayPlacental > histogram.data 2>&1
    #	real    8m15.623s

    #	create plot of histogram:

    cat << '_EOF_' | gnuplot > histo.png
set terminal png small color x000000 xffffff xc000ff x66ff66 xffff00 x00ffff
set size 1.4, 0.8
set key left box
set grid noxtics
set grid ytics
set title " Human Hg19 Histogram phyloP46wayPlacental track"
set xlabel " phyloP46wayPlacental score"
set ylabel " Relative Frequency"
set y2label " Cumulative Relative Frequency (CRF)"
set y2range [0:1]
set y2tics
set yrange [0:0.03]
set xrange [-2.5:2.5]

plot "histogram.data" using 2:5 title " RelFreq" with impulses, \
        "histogram.data" using 2:7 axes x1y2 title " CRF" with lines
'_EOF_'
    #	<< happy emacs

    display histo.png &

#########################################################################
# LASTZ Zebrafish DanRer6 (DONE - 2009-07-08,10 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzDanRer6.2009-07-08
    cd /hive/data/genomes/hg19/bed/lastzDanRer6.2009-07-08

    cat << '_EOF_' > DEF
# human vs X. zebrafish
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Zebrafish danRer6
SEQ2_DIR=/scratch/data/danRer6/danRer6.2bit
SEQ2_LEN=/scratch/data/danRer6/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LAP=0
SEQ2_LIMIT=40

BASE=/hive/data/genomes/hg19/bed/lastzDanRer6.2009-07-08
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    1678m17.827s
    #	failed during the chain step due to encodek cluster problems
    #	finish that manually, then:
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-continue=chainMerge > chainMerge.log 2>&1 &
    #	real    167m6.930s
    cat fb.hg19.chainDanRer6Link.txt 
    #	88391631 bases of 2897316137 (3.051%) in intersection

    #	running the swap - DONE - 2009-06-02
    mkdir /hive/data/genomes/danRer6/bed/blastz.hg19.swap
    cd /hive/data/genomes/danRer6/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzDanRer6.2009-07-08/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-swap > swap.log 2>&1 &
    #	real    183m21.102s
    cat fb.danRer6.chainHg19Link.txt 
    #	96424507 bases of 1506896106 (6.399%) in intersection

##############################################################################
# LASTZ Elephant LoxAfr3 (DONE - 2009-07-21,23 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzLoxAfr3.2009-07-21
    cd /hive/data/genomes/hg19/bed/lastzLoxAfr3.2009-07-21

    cat << '_EOF_' > DEF
# Human vs. Elephant
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Elephant
SEQ2_DIR=/scratch/data/loxAfr3/loxAfr3.2bit
SEQ2_LEN=/scratch/data/loxAfr3/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LIMIT=50
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzLoxAfr3.2009-07-21
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    317m32.664s
    #	broken when it went to chaining on encodek, finish the chain then:
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-continue=chainMerge > chainMerge.log 2>&1 &
    #	real    217m25.159s

    # time about 3h23m
    cat fb.hg19.chainLoxAfr3Link.txt 
    #	1351200080 bases of 2897316137 (46.636%) in intersection

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-syntenicNet -continue=syntenicNet -stop=syntenicNet \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	> synNet.log 2>&1 &
    #	real    32m40.554s

    time doRecipBest.pl -buildDir=`pwd` hg19 loxAfr3 > rbest.log 2>&1
    #	real    184m3.435s

    mkdir /hive/data/genomes/loxAfr3/bed/blastz.hg19.swap
    cd /hive/data/genomes/loxAfr3/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzLoxAfr3.2009-07-21/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-swap > swap.log 2>&1 &
    #	real    220m16.839s
    cat fb.loxAfr3.chainHg19Link.txt 
    #	1323201500 bases of 3118565340 (42.430%) in intersection

##############################################################################
# TRANSMAP vertebrate.2009-07-01 build  (2009-07-21 markd)

vertebrate-wide transMap alignments were built  Tracks are created and loaded
by a single Makefile. This is available from:
   svn+ssh://hgwdev.cse.ucsc.edu/projects/compbio/usr/markd/svn/projs/transMap/tags/vertebrate.2009-07-01

see doc/builds.txt for specific details.

############################################################################
# AGILENT PROBES LIFTED FROM HG18 (DONE, 2009-07-28 Andy)

ssh hgwdev
bash
mkdir /hive/data/genomes/hg19/bed/agilentProbes
cd /hive/data/genomes/hg19/bed/agilentProbes
for table in `echo show tables like \'agilent%\' | hgsql hg18 | tail -n +2 | grep -v Probe`; do
    echo $table; echo "select * from $table" | hgsql hg18 | \
        tail -n +2 | cut -f2- > ${table}.hg18.bed; liftOver ${table}.hg18.bed \
          /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz ${table}.hg19.{bed,unmapped};
    hgLoadBed hg19 $table ${table}.hg19.bed; 
    echo done with $table; 
done
for unmap in *.unmapped; do
   table=${unmap%.hg19.unmapped}
   grep Deleted -A1 $unmap | grep -v Deleted | grep -v "^--" > agilentProbesHg18Unmapped/${table}.deleted.bed
   grep Split -A1 $unmap | grep -v Split | grep -v "^--" > agilentProbesHg18Unmapped/${table}.split.bed
   grep Partially -A1 $unmap | grep -v Partially | grep -v "^--" > agilentProbesHg18Unmapped/${table}.partiallyDeleted.bed
done
find agilentProbesHg18Unmapped/ -size 0b | xargs rm
rm *hg18.bed *.unmapped bed.tab
gzip *.bed
tar cfz agilentProbesHg18Unmapped.tar.gz agilentProbesHg18Unmapped
cd /usr/local/apache/htdocs/goldenPath/hg19
mkdir agilentProbes
cd agilentProbes/
ln -s /hive/data/genomes/hg19/bed/agilentProbes/agilentProbesHg18Unmapped beds
ln -s /hive/data/genomes/hg19/bed/agilentProbes/agilentProbesHg18Unmapped.tar.gz

##############################################################################
# LASTZ Tetraodon TetNig2 (DONE - 2009-08-10,11 - Hiram)
    #	This is the incorrect date/time stamp on this directory,
    #	it should be 2009-08-10
    mkdir /hive/data/genomes/hg19/bed/lastzTetNig2.2009-10-10
    cd /hive/data/genomes/hg19/bed/lastzTetNig2.2009-10-10

    cat << '_EOF_' > DEF
# human vs tetraodon
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_M=50
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Tetraodon TetNig2 - single chunk big enough to single largest item
SEQ2_DIR=/scratch/data/tetNig2/tetNig2.2bit
SEQ2_LEN=/scratch/data/tetNig2/chrom.sizes
SEQ2_CTGDIR=/scratch/data/tetNig2/tetNig2.contigs.2bit
SEQ2_CTGLEN=/scratch/data/tetNig2/tetNig2.contigs.sizes
SEQ2_LIFT=/scratch/data/tetNig2/tetNig2.contigs.lift
SEQ2_CHUNK=20000000
SEQ2_LAP=0
SEQ2_LIMIT=50

BASE=/hive/data/genomes/hg19/bed/lastzTetNig2.2009-10-10
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    220m36.068s
    #	forgot the qRepeats for tetNig2
    rm axtChain/hg19.tetNig2.net
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-continue=load -qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	> load.log 2>&1 &
    #	real    5m53.096s
    cat fb.hg19.chainTetNig2Link.txt 
    #	49611132 bases of 2897316137 (1.712%) in intersection

    #	running the swap
    mkdir /hive/data/genomes/tetNig2/bed/blastz.hg19.swap
    cd /hive/data/genomes/tetNig2/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzTetNig2.2009-10-10/DEF \
	-qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-swap > swap.log 2>&1 &
    #	real    13m21.591s
    #	forgot the qRepeats for tetNig2
    rm axtChain/tetNig2.hg19.net
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzTetNig2.2009-10-10/DEF \
	-continue=load -qRepeats=windowmaskerSdust \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-swap > load.log 2>&1 &
    #	real    4m7.559s
    cat fb.tetNig2.chainHg19Link.txt 
    #	42910930 bases of 302314788 (14.194%) in intersection


##############################################################################
# dbSNP BUILD 130 - PROVISIONAL REMAPPING TO BUILD 37 (DONE 8/28/09 angie)
    # /hive/data/outside/dbSNP/130/ was already set up during the hg18 run --
    # just add hg19 coord files and go from there.
    cd /hive/data/outside/dbSNP/130/human/data
    alias wg wget --timestamping
    set ftpSnpDb = ftp://ftp.ncbi.nih.gov/snp/organisms/human_9606/misc/exchange
    # These are provisional files in an ad-hoc format.
    wg $ftpSnpDb/README.txt
    wg $ftpSnpDb/Remap_36_3_37_1.info
    wg $ftpSnpDb/Remap_36_3_37_1.txt.gz
    mv README.txt Remap_36_3_37_1_README
    zcat Remap_36_3_37_1.txt.gz | wc -l
#18823990

    # Use the remapping to transform ../ucscNcbiSnp.bed into one for hg19.
    # Useful columns, 1-based: 1=ID, 3=oldChr, 4=oldStart, 5=oldEnd,
    # 10=newChr, 11=newStart, 12=newEnd, 13=newLocType, 14=newWeight, 16=newStrand
    # For mappings to chr*_random, oldStart and oldEnd are empty -- skip.
    # Sort both hg18 snp file and remap file by {rsID,chr,start} to keep them in sync.
    mkdir /hive/data/outside/dbSNP/130/human/hg19
    cd /hive/data/outside/dbSNP/130/human/hg19
    sort -k4n,4n -k1,1 -k2n,2n ../ucscNcbiSnp.bed > /data/tmp/hg18.ucscNcbiSnp.idSorted.bed
    zcat ../data/Remap_36_3_37_1.txt.gz \
    | sort -t "	" -k1n,1n -k3,3 -k4n,4n \
      > /data/tmp/Remap_36_3_37_1.txt
    perl -we \
      'use strict; \
       sub nextMap { \
         my ($rsId, undef, $oChr, $oStart, $oEnd, $nChr, $nStart, $nEnd, \
             $nLocType, $nWt, $nRef, $nStr);\
         do { \
           ($rsId, undef, $oChr, $oStart, $oEnd, undef,undef,undef,undef, \
               $nChr, $nStart, $nEnd, $nLocType, $nWt, $nRef, $nStr) = split("\t", <>); \
           if (defined $nStr) { \
             chomp $nStr; $nStr =~ tr/+-/01/; $oChr = "chr$oChr";  $nChr = "chr$nChr"; \
           } \
           $oStart--;  $oEnd--;  $nStart--;  $nEnd--;  # Yep. 0-based closed vs 1-based closed \
         } while (defined $nStr && ($oEnd < 0 || $nChr eq "chrUn")); \
         return ($rsId, $oChr, $oStart, $oEnd, $nChr, $nStart, $nEnd, \
                 $nLocType, $nWt, $nRef, $nStr); \
       } # nextMap \
       my ($rsId, $oChr, $oStart, $oEnd, $nChr, $nStart, $nEnd, $nLocType, $nWt, $nRef, $nStr) = \
         &nextMap(); \
       my ($rCount, $oCount, $tCount) = 0; \
       open(my $oldF, "/data/tmp/hg18.ucscNcbiSnp.idSorted.bed") || die; \
       while (my ($chr, $s, $e, $id, $str, $rn,$obs,$mt,$cn,$vn,$ah,$ahse,$fc,$lt,$wt) = \
              split("\t", <$oldF>)) { \
         my $thisRCount = 0; \
         while (defined $oChr && $chr eq $oChr && $s == $oStart && $e == $oEnd && $id == $rsId) { \
           print join("\t", $nChr,$nStart,$nEnd,$id,$nStr,$nRef,$obs,$mt,$cn,$vn,$ah,$ahse,$fc, \
                            $nLocType,$nWt,$nStart) \
                      . "\n"; \
           ($rsId, $oChr, $oStart, $oEnd, $nChr, $nStart, $nEnd, $nLocType, $nWt, $nRef, $nStr) = \
             &nextMap(); \
           $thisRCount++; \
         } \
         if (defined $rsId && $id > $rsId) {warn "Slipped a cog"; last;} \
         $tCount += $thisRCount; \
         $rCount++ if ($thisRCount > 0); \
         $oCount++; \
       } \
       close($oldF);  print STDERR "Replaced $rCount of $oCount inputs ($tCount outputs).\n";' \
      /data/tmp/Remap_36_3_37_1.txt \
    | sort -k1,1 -k2n,2n -k4,4 \
    > /data/tmp/hg19.ucscNcbiSnp.bed
#Replaced 18693260 of 19189750 inputs (18697579 outputs).
#504.562u 27.037s 8:59.57 98.5%  0+0k 0+0io 0pf+0w
    wc -l /data/tmp/hg19.ucscNcbiSnp.bed
#  18697579 /data/tmp/hg19.ucscNcbiSnp.bed

    # Drum roll please... translate NCBI's encoding into UCSC's, and
    # perform a bunch of checks.  This is where developer involvement
    # is most likely as NCBI extends the encodings used in dbSNP.
    cd /hive/data/outside/dbSNP/130/human/hg19
    snpNcbiToUcsc /data/tmp/hg19.ucscNcbiSnp.bed /hive/data/genomes/hg19/hg19.2bit \
      -1000GenomesRsIds=../data/1000GenomesRsIds.txt snp130
#spaces stripped from observed:
#chr12   6093134 6093134 rs41402545
#Line 8049395 of /data/tmp/hg19.ucscNcbiSnp.bed: Encountered something that doesn't fit observedMixedFormat: GCAACTTCA
#count of snps with weight  0 = 0
#count of snps with weight  1 = 17042465
#count of snps with weight  2 = 345274
#count of snps with weight  3 = 1017906
#count of snps with weight 10 = 291934
#Skipped 1496 snp mappings due to errors -- see snp130Errors.bed
#146.837u 9.867s 4:21.63 59.8%   0+0k 0+0io 0pf+0w
    # Comparable to hg18.snp130, with some losses due to coord translation, loss of _randoms,
    # and 1496 errors (new locType or refNCBI inconsistent with new size).
    expr 18697579 - 291934 - 1496
#18404149

    # Move hg19.ucscNcbiSnp.bed from fast tmp to slow (today) hive:
    gzip /data/tmp/hg19.ucscNcbiSnp.bed
    mv /data/tmp/hg19.ucscNcbiSnp.bed.gz hg19.ucscNcbiSnp.bed.gz

    # Will try not reuse hg18.snp130's giant 18G fasta file, not duplicate.

    # Load up main track tables.
    cd /hive/data/outside/dbSNP/130/human/hg19
    hgLoadBed -tab -tmpDir=/data/tmp -allowStartEqualEnd \
      hg19 snp130 -sqlTable=snp130.sql snp130.bed
#Loaded 18404149 elements of size 17
#115.086u 21.663s 2:32:09.98 1.4%        0+0k 0+0io 1pf+0w
#that is freakishly long -- lots happening today w/db move, hive recovery,...
    hgLoadBed -tab -onServer -tmpDir=/data/tmp -allowStartEqualEnd \
      hg19 snp130Exceptions -sqlTable=$HOME/kent/src/hg/lib/snp125Exceptions.sql -renameSqlTable \
      snp130Exceptions.bed
#Loaded 1982828 elements of size 5
#10.500u 0.851s 1:13.42 15.4%    0+0k 0+0io 0pf+0w
    hgLoadSqlTab hg19 snp130ExceptionDesc ~/kent/src/hg/lib/snp125ExceptionDesc.sql \
      snp130ExceptionDesc.tab
    # Load up sequences *from hg18 file*:
    hgLoadSqlTab hg19 snp130Seq ~/kent/src/hg/lib/snpSeq.sql ../snp130Seq.tab

    # Put in a link where one would expect to find the track build dir...
    ln -s /hive/data/outside/dbSNP/130/human/hg19 /hive/data/genomes/hg19/bed/snp130

    # Look at the breakdown of exception categories:
    cd /hive/data/outside/dbSNP/130/human/hg19
    cut -f 5 snp130Exceptions.bed | sort | uniq -c | sort -nr
#1350217 MultipleAlignments
# 495981 ObservedMismatch
#  37603 ObservedTooLong
#  26855 SingleClassTriAllelic
#  24443 FlankMismatchGenomeShorter
#  17927 SingleClassLongerSpan
#  13685 SingleClassZeroSpan
#   6238 FlankMismatchGenomeLonger
#   3016 DuplicateObserved
#   2851 SingleClassQuadAllelic
#   1777 MixedObserved
#   1264 NamedDeletionZeroSpan
#    508 FlankMismatchGenomeEqual
#    329 NamedInsertionNonzeroSpan
#    121 ObservedContainsIupac
#     11 RefAlleleMismatch
#      2 ObservedWrongFormat

#TODO: go through those above (esp snp130Errors.bed) and send some bug reports to dbSNP.


##############################################################################
# ORTHOLOGOUS ALLELES IN CHIMP AND MACAQUE FOR SNP130 (DONE 8/31/09 angie)
    mkdir /hive/data/genomes/hg19/bed/snp130Ortho
    cd /hive/data/genomes/hg19/bed/snp130Ortho

    # Following Heather's lead in snp126orthos, filter SNPs to to keep
    # only those with class=single, length=1, chrom!~random;
    # Exclude those with exceptions MultipleAlignments,
    # SingleClassTriAllelic or SingleClassQuadAllelic.
    # Unlike snp masking, we do not filter for weight -- don't know why.
    awk '$5 ~ /^MultipleAlignments|SingleClassTriAllelic|SingleClassQuadAllelic/ {print $4;}' \
      /hive/data/outside/dbSNP/130/human/hg19/snp130Exceptions.bed \
    | sort -u \
      > snp130ExcludeIds.txt
    awk '$3-$2 == 1 && $1 !~ /_random/ && $11 == "single" {print;}' \
      /hive/data/outside/dbSNP/130/human/hg19/snp130.bed \
    | grep -vFwf snp130ExcludeIds.txt \
      > snp130Simple.bed
#203.193u 9.197s 2:57.40 119.7%  0+0k 0+0io 0pf+0w
    wc -l snp130Simple.bed
#12278514 snp130Simple.bed

    # Glom all human info that we need for the final table onto the
    # name, to sneak it through liftOver: rsId|chr|start|end|obs|ref|strand
    awk 'BEGIN{OFS="\t";} \
        {print $1, $2, $3, \
               $4 "|" $1 "|" $2 "|" $3 "|" $9 "|" $8 "|" $6, \
               0, $6;}' \
      snp130Simple.bed > snp130ForLiftOver.bed
    # Map coords to chimp using liftOver.
    # I don't know why chimp took so much longer than macaque... the
    # chimp .over has fewer chains and fewer bytes than the macaque .over.
    mkdir run.liftOChimp
    cd run.liftOChimp
    mkdir split out
    splitFile ../snp130ForLiftOver.bed 25000 split/chunk
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToPanTro2.over.chain.gz \
        \{check out exists out/panTro2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    ssh swarm
    cd /hive/data/genomes/hg19/bed/snp130Ortho/run.liftOChimp
    para make jobList
#Completed: 492 of 492 jobs
#CPU time in finished jobs:      51793s     863.22m    14.39h    0.60d  0.002 y
#IO & Wait Time:                  3825s      63.75m     1.06h    0.04d  0.000 y
#Average job time:                 113s       1.88m     0.03h    0.00d
#Longest finished job:             286s       4.77m     0.08h    0.00d
#Submission to last job:           300s       5.00m     0.08h    0.00d

    # Map coords to orangutan using liftOver.
    mkdir ../run.liftOPon
    cd ../run.liftOPon
    mkdir out
    ln -s ../run.liftOChimp/split .
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToPonAbe2.over.chain.gz \
        \{check out exists out/ponAbe2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    para make jobList
#Completed: 492 of 492 jobs
#CPU time in finished jobs:     125656s    2094.26m    34.90h    1.45d  0.004 y
#IO & Wait Time:                  5413s      90.22m     1.50h    0.06d  0.000 y
#Average job time:                 266s       4.44m     0.07h    0.00d
#Longest finished job:             646s      10.77m     0.18h    0.01d
#Submission to last job:           649s      10.82m     0.18h    0.01d

    # Map coords to macaque using liftOver.
    mkdir ../run.liftOMac
    cd ../run.liftOMac
    mkdir out
    ln -s ../run.liftOChimp/split .
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToRheMac2.over.chain.gz \
        \{check out exists out/rheMac2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    para make jobList
#Completed: 492 of 492 jobs
#CPU time in finished jobs:     161612s    2693.54m    44.89h    1.87d  0.005 y
#IO & Wait Time:                  6218s     103.63m     1.73h    0.07d  0.000 y
#Average job time:                 341s       5.69m     0.09h    0.00d
#Longest finished job:             727s      12.12m     0.20h    0.01d
#Submission to last job:           739s      12.32m     0.21h    0.01d

    cd /hive/data/genomes/hg19/bed/snp130Ortho
    # Concatenate the chimp results, sorting by chimp pos in order to
    # efficiently access 2bit sequence in getOrthoSeq.  The output of
    # that is then sorted by the glommed human info field, so that we
    # can use join to combine chimp and macaque results in the next step.
    # Ditto for macaque and orangutan.  Each command pipe takes ~5 minutes:
    sort -k1,1 -k2n,2n run.liftOChimp/out/panTro2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/panTro2/panTro2.2bit \
    | sort > panTro2.orthoGlom.txt
    sort -k1,1 -k2n,2n run.liftOPon/out/ponAbe2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/ponAbe2/ponAbe2.2bit \
    | sort > ponAbe2.orthoGlom.txt
    sort -k1,1 -k2n,2n run.liftOMac/out/rheMac2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/rheMac2/rheMac2.2bit \
    | sort > rheMac2.orthoGlom.txt
    wc -l panTro2.orthoGlom.txt ponAbe2.orthoGlom.txt rheMac2.orthoGlom.txt
#  11428526 panTro2.orthoGlom.txt
#  10861969 ponAbe2.orthoGlom.txt
#   9694237 rheMac2.orthoGlom.txt

    # Use the glommed name field as a key to join up chimp and macaque
    # allele data.  Include glommed name from both files because if only
    # file 2 has a line for the key in 2.1, then 1.1 is empty.  Then plop
    # in the orthoGlom fields from each file, which are in the same order
    # as the chimp and macaque columns of snp130OrthoPanTro2RheMac2.
    join -o '1.1 2.1 1.2 1.3 1.4 1.5 1.6 2.2 2.3 2.4 2.5 2.6' \
      -a 1 -a 2 -e '?' \
      panTro2.orthoGlom.txt ponAbe2.orthoGlom.txt \
    | awk '{if ($1 != "?") { print $1, $3,$4,$5,$6,$7,$8,$9,$10,$11,$12; } \
            else           { print $2, $3,$4,$5,$6,$7,$8,$9,$10,$11,$12; }}' \
      > tmp.txt
    join -o '1.1 2.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 2.2 2.3 2.4 2.5 2.6' \
      -a 1 -a 2 -e '?' \
      tmp.txt rheMac2.orthoGlom.txt \
    | perl -wpe 'chomp; \
        ($glom12, $glom3, $o1Chr, $o1Start, $o1End, $o1Al, $o1Strand, \
         $o2Chr, $o2Start, $o2End, $o2Al, $o2Strand, \
         $o3Chr, $o3Start, $o3End, $o3Al, $o3Strand) = split; \
        $glomKey = ($glom12 ne "?") ? $glom12 : $glom3; \
        ($rsId, $hChr, $hStart, $hEnd, $hObs, $hAl, $hStrand) = \
          split(/\|/, $glomKey); \
        $o1Start =~ s/^\?$/0/;  $o2Start =~ s/^\?$/0/;  $o3Start =~ s/^\?$/0/; \
        $o1End   =~ s/^\?$/0/;  $o2End   =~ s/^\?$/0/;  $o3End   =~ s/^\?$/0/; \
        print join("\t", $hChr, $hStart, $hEnd, $rsId, $hObs, $hAl, $hStrand, \
                         $o1Chr, $o1Start, $o1End, $o1Al, $o1Strand, \
                         $o2Chr, $o2Start, $o2End, $o2Al, $o2Strand, \
                         $o3Chr, $o3Start, $o3End, $o3Al, $o3Strand) . "\n"; \
        s/^.*$//;' \
    | sort -k1,1 -k2n,2n > snp130OrthoPt2Pa2Rm2.bed
#304.434u 27.118s 4:31.30 122.2% 0+0k 0+0io 0pf+0w
    wc -l snp130OrthoPt2Pa2Rm2.bed
#11876029 snp130OrthoPt2Pa2Rm2.bed

    cd /hive/data/genomes/hg19/bed/snp130Ortho
    hgLoadBed -tab -onServer -tmpDir=/data/tmp -renameSqlTable \
      -sqlTable=$HOME/kent/src/hg/lib/snpOrthoPanPonRhe.sql \
      hg19 snp130OrthoPt2Pa2Rm2 snp130OrthoPt2Pa2Rm2.bed
#Loaded 11876029 elements of size 22
#75.442u 8.828s 9:50.27 14.2%    0+0k 0+0io 0pf+0w

    # Cleanup fileserver:
    cd /hive/data/genomes/hg19/bed/snp130Ortho
    gzip snp130Simple.bed snp130ExcludeIds.txt snp130ForLiftOver.bed &
    rm -r run*/split tmp.txt *.orthoGlom.txt

##############################################################################
# LASTZ Rabbit OryCun2 (DONE - 2009-08-12 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzOryCun2.2009-08-12
    cd /hive/data/genomes/hg19/bed/lastzOryCun2.2009-08-12

    cat << '_EOF_' > DEF
# Human vs. Rabbit
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Rabbit at chunk 20,000,000 all but 36 contigs can fit in a single job
SEQ2_DIR=/scratch/data/oryCun2/oryCun2.2bit
SEQ2_LEN=/scratch/data/oryCun2/chrom.sizes
SEQ2_CTGDIR=/scratch/data/oryCun2/oryCun2.contigs.2bit
SEQ2_CTGLEN=/scratch/data/oryCun2/oryCun2.contigs.sizes
SEQ2_LIFT=/hive/data/genomes/oryCun2/contigs/oryCun2.contigs.lift
SEQ2_CHUNK=20000000
SEQ2_LIMIT=400
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzOryCun2.2009-08-12
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    516m41.981s
    cat fb.hg19.chainOryCun2Link.txt 
    #	1283994337 bases of 2897316137 (44.317%) in intersection
    #	should have run syntenicNet in that first run
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-continue=syntenicNet -syntenicNet > syntenicNet.log 2>&1 &
    #	about 1 hour

    mkdir /hive/data/genomes/oryCun2/bed/blastz.hg19.swap
    cd /hive/data/genomes/oryCun2/bed/blastz.hg19.swap
    
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzOryCun2.2009-08-12/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-swap -syntenicNet > swap.log 2>&1 &
    #	real    176m35.932s
    cat fb.oryCun2.chainHg19Link.txt 
    #	1260477501 bases of 2604023284 (48.405%) in intersection

##############################################################################
# running syntenicNet on CavPor3 lastz (DONE - 2009-08-27 - Hiram)
    cd /hive/data/genomes/hg19/bed/lastzCavPor3.2009-06-04
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-continue=syntenicNet -syntenicNet > syntenicNet.log 2>&1 &
    #	about 44 minutes

##############################################################################
# loading the lastz tables on cavPor3 - (DONE - 2009-08-28 - Hiram)
    # the chain.tab and link.tab files are left over from the failed load
    cd  /hive/data/genomes/cavPor3/bed/blastz.hg19.swap/axtChain

    #	find out their sizes, average and total:
    awk '{print length($0)}' chain.tab | ave stdin
Q1 92.000000 median 93.000000 Q3 96.000000
average 93.651267
min 64.000000 max 109.000000
count 27186468
total 2546047186.000000
    awk '{print length($0)}' link.tab | ave stdin
Q1 45.000000 median 47.000000 Q3 48.000000
average 46.731871
min 22.000000 max 52.000000
count 240602108
total 11243786622.000000

    cat << '_EOF_' > chainHg19Link.sql
CREATE TABLE chainHg19Link (
  bin smallint(5) unsigned NOT NULL default 0,
  tName varchar(255) NOT NULL default '',
  tStart int(10) unsigned NOT NULL default 0,
  tEnd int(10) unsigned NOT NULL default 0,
  qStart int(10) unsigned NOT NULL default 0,
  chainId int(10) unsigned NOT NULL default 0,
  KEY tName (tName(13),bin),
  KEY chainId (chainId)
) ENGINE=MyISAM max_rows=241000000 avg_row_length=50 pack_keys=1 CHARSET=latin1;
'_EOF_'
    # << happy emacs
    hgsql cavPor3 < chainHg19Link.sql

    time hgsql -e \
      'load data local infile "link.tab" into table chainHg19Link;' cavPor3
    #	real    405m15.956s

    cd  /hive/data/genomes/cavPor3/bed/blastz.hg19.swap/axtChain

    #	and the net tracks were not loaded:
    time netClass -verbose=0 -noAr noClass.net cavPor3 hg19 cavPor3.hg19.net
    #	real    40m25.078s

    netFilter -minGap=10 cavPor3.hg19.net \
	| hgLoadNet -verbose=0 cavPor3 netHg19 stdin
    # real    33m24.972s (plus the featureBits below)

    featureBits cavPor3 chainHg19Link > fb.cavPor3.chainHg19Link.txt 2>&1
    cat fb.cavPor3.chainHg19Link.txt
    #	1279572660 bases of 2663369733 (48.043%) in intersection

##############################################################################
# DBSNP CODING ANNOTATIONS (DONE 10/12/10 angie)
# Updated 10/12/10 using rebuilt hg18 snp130CodingDbSnp.bed w/corrected coords.
# Originally done 9/1/09

    # Repeat the coord-remapping performed for snp130 on the hg18 coding anno table.
    cd /hive/data/outside/dbSNP/130/human/hg19
    sed -re 's/\trs([0-9]+)\t/\t\1\t/' ../snp130CodingDbSnp.bed \
    | sort -k4n,4n -k1,1 -k2n,2n > /data/tmp/hg18.snp130Coding.idSorted.bed
    # reuse /data/tmp/Remap_36_3_37_1.txt mapping file created for snp130 above,
    # but first translate its coords (1-based fully-closed with 2-base-long insertions)
    # into ours (0-based half-open with 0-base-long insertions) and discard incompletes.
    perl -we \
      'while (my ($rsId, undef, $oChr, $oStart, $oEnd, $oLocType, undef,undef,undef, \
                                $nChr, $nStart, $nEnd, $nLocType) = split("\t", <>)) { \
         next if ($oStart eq "" || $nStart eq ""); \
         $oChr = "chr$oChr";  $nChr = "chr$nChr"; \
         # 2-base-long insertion (loc_type==3) -> 0-base-long: \
         if ($oLocType == 3) { $oEnd--; } else { $oStart--; } \
         if ($nLocType == 3) { $nEnd--; } else { $nStart--; } \
         print join("\t", $rsId, $oChr, $oStart, $oEnd, $nChr, $nStart, $nEnd) . "\n"; \
       }' /data/tmp/Remap_36_3_37_1.txt \
      > /data/tmp/Remap_36_3_37_1_ucscCoords.txt
    # Apply the cleaned-up mapping to id-sorted hg18 snp130CodingDbSnp:
    perl -we \
      'use strict; \
       my ($rsId, $oChr, $oStart, $oEnd, $nChr, $nStart, $nEnd) = split("\t", <>); \
       my ($rCount, $oCount, $tCount) = 0; \
       open(my $oldF, "/data/tmp/hg18.snp130Coding.idSorted.bed") || die; \
       while (my ($chr, $s, $e, $id, $tx, $frm, $alCount, $funcs, $als, $codons, $peps) = \
              split("\t", <$oldF>)) { \
         my $thisRCount = 0; \
         while (defined $rsId && $rsId < $id) { \
           ($rsId, $oChr, $oStart, $oEnd, $nChr, $nStart, $nEnd) = split("\t", <>); \
         } \
         while (defined $oChr && $chr eq $oChr && $s == $oStart && $e == $oEnd && $id == $rsId) { \
           chomp $nEnd; \
           print join("\t", $nChr, $nStart, $nEnd, "rs$id", $tx, $frm, \
                            $alCount, $funcs, $als, $codons, $peps) unless $nEnd < $nStart; \
           ($rsId, $oChr, $oStart, $oEnd, $nChr, $nStart, $nEnd) = split("\t", <>); \
           $thisRCount++; \
         } \
         $tCount += $thisRCount; \
         $rCount++ if ($thisRCount > 0); \
         $oCount++; \
       } \
       close($oldF);  print STDERR "Replaced $rCount of $oCount inputs ($tCount outputs).\n";' \
      /data/tmp/Remap_36_3_37_1_ucscCoords.txt \
    | sort -k1,1 -k2n,2n -k4,4 \
    > /data/tmp/hg19.snp130Coding.bed
#Replaced 197921 of 279815 inputs (198493 outputs).
#35.486u 1.515s 0:36.70 100.7%   0+0k 0+0io 0pf+0w
   hgLoadBed hg19 snp130CodingDbSnp -sqlTable=$HOME/kent/src/hg/lib/snp125Coding.sql \
      -renameSqlTable -tab -notItemRgb -allowStartEqualEnd \
      /data/tmp/hg19.snp130Coding.bed
#Loaded 198459 elements of size 11
    # A bit fewer than reported 198493 above, but we ditched a few with $nEnd < $nStart
    # (corresponding SNPs ended up in snp130Errors.bed not snp130.bed anyway).
    mv /data/tmp/hg19.snp130Coding.bed hg19.snp130CodingDbSnp.bed

############################################################################
# TRANSMAP vertebrate.2009-09-13 build  (2009-09-20 markd)

vertebrate-wide transMap alignments were built  Tracks are created and loaded
by a single Makefile. This is available from:
   svn+ssh://hgwdev.cse.ucsc.edu/projects/compbio/usr/markd/svn/projs/transMap/tags/vertebrate.2009-09-13

see doc/builds.txt for specific details.
############################################################################
# BURGE	LAB DATA MAPPED WITH GEMMAPPER. PROVIDED BY THOMAS DERRIEN FROM RODERIC
# GUIGO'S LAB AT CRG. (E-MAIL: thomas.derrien@crg.es). Data received on
# 09/14/09. 
# (hartera, 2009-09-28, DONE) 
# 2009-12-14, hartera. Set cdsStart = cdsEnd = 0. Moved track data directory to 
# /hive/data/genomes/hg18/bed.
# 2010-01-04, hartera. Change the data to BED format and re-loaded tables. BED
# is more appropriate for this data type.
# The data is too dense in places (feedback from QA) so it would be more
# appropriate to have a Signal track as for the ENCODE RNA-seq data tracks. 
# 2010-02-09, hartera. Create bedGraph Signal subtracks for each tissue/cell
# using reads/per million mapped reads as the data value.
# 2010-02-17, hartera. Updated trackDb.ra entry to include views.
# 2010-02-18, hartera. Loaded the bedGraph tables for the Raw Signal
# subtracks.
# 2010-05-15 and 2010-05-16, hartera. Re-created the Signal subtracks using 
# the -bed12 option of bedItemOverlapCount so that blocks are used. 
 
   mkdir /hive/groups/gencode/browser/hg19/burgeRnaSeqGemMapperAlign
   cd /hive/groups/gencode/browser/hg19/burgeRnaSeqGemMapperAlign

# Added the statements below to a script so that it can be run to fetch
# all the sequences.

wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325476_brain_HCT168_hg18.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325477_liver_HCT169_hg18.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325478_heart_HCT170_hg18.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325479_skelMuscle_HCT171_hg18.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325480_colon_hg18.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325481_adipose_hg18.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325482_testes_hg18.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325483_lymphNode_hg18.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325484_HCT204_bt474_.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325485_HCT205_HME_.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325486_HCT202_s2468_.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325487_HCT203_s2468.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325488_HCT206_s2468_.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
wget --timestamping "http://genome.imim.es/~tderrien/Gencode/For_Rachel/GFF_Burge_32bp_Gencode3/GSM325489_HCT207_s2468_.fa.GEM_index.GRCh37.gencode.v3.annotation.GRCh37.exon.gtf.length_32.gff.gz"
  
   # Load this data into tables for hg19.
   # Unzip the files:
   gunzip *.gff.gz 
   # Create a file with the list of file names and tissues.
   ls *.gff > burgeDataFiles.txt
   GSM325486_HCT202_s2468	breast
   GSM325487_HCT203_s2468	MCF-7
   GSM325488_HCT206_s2468	MB435 
   GSM325489_HCT207_s2468	T47D
   # Did not map these two as they are not 32 bp. 
   GSM325490_brain_s1368	MAQC	mixed human brain tissue/cell lines
   GSM325491_UHR_s247		MAQC_UHR mixed human cell lines
   # Edit the file above to add a tab separation between file name and tissue
   # name. Then remove the "read_name: " from the last field in each 
   # file otherwise it gets included in the name and load the data into hg18.
   # Write a script to do this: 
cat << '_EOF_' > formatAndLoadData
#!/bin/bash -e
   
# Assign variables
# Tab-separated file of file names and tissue/cell line names
DATAFILES=$1
# track name used as prefix for subtracks
TRACK=$2
# database
DATABASE=$3

cat $DATAFILES | while read file tissue; do
    subTrack=`echo $TRACK$tissue`
    echo $subTrack   
    sed -e 's/read_name:\s//' $file > ${subTrack}.gff
    ldHgGene -exon=read $DATABASE ${subTrack} ${subTrack}.gff 
done
'_EOF_'
   chmod +x formatAndLoadData
   ./formatAndLoadData burgeDataFiles.txt burgeRnaSeqGemMapperAlign hg19 \
     >& load.log &
   # Took about 2 hours to load the tables.
   # Copy trackDb entry in hg18 trackDb.ra to
   # ccds/trunk/gencode/browser/trackDb/human/hg19/trackDb.ra

   # 2009-12-14, Need to change cdsStart = cdsEnd = 0 in the tables as this
   # data should have no CDS defined. Currently cdsStart = cdsEnd = txEnd. 
   cd /hive/groups/gencode/browser/hg19/burgeRnaSeqGemMapperAlign
   hgsql -Ne 'show tables like "burge%";' hg19 > burgeTables
   foreach t (`cat burgeTables`)
      echo $t
      hgsql -e "update $t set cdsStart = 0;" hg19
      hgsql -e "update $t set cdsEnd = 0;" hg19
   end
   # Then move data to directory in hg19 genome bed directory
   cd /hive/data/genomes/hg19/bed
   mv /hive/groups/gencode/browser/hg19/burgeRnaSeqGemMapperAlign ./

   # 2010-01-04 Change the data to BED format. For genePred format, 
   # there is always a track configuration added for colouring tracks by
   # genomic codons which does not make sense for this data. Also, BED is
   # more appropriate for this data type.
   cd /hive/data/genomes/hg19/bed/burgeRnaSeqGemMapperAlign
   # Convert gff to genePred and then genePred to BED, drop old table and
   # then load database with BED format data. Need to fix the cdStart and
   # cdsEnd fields to be 0. 
   foreach f (`ls burgeRnaSeqGemMapperAlign*.gff`)
     echo $f >> bed.log
     set g=$f:r
     echo $g
     ldHgGene -exon=read -nobin -out=${g}.gp hg19 $g $f >>& bed.log
     awk 'BEGIN {FS="\t"} {OFS="\t"} {print $1,$2,$3,$4,$5,0,0,$8,$9,$10}' \
         ${g}.gp > ${g}Fixed.gp 
     genePredToBed ${g}Fixed.gp > ${g}.bed 
     echo "Dropping table $g"
     hgsql -e "drop table ${g};" hg19
     hgLoadBed hg19 $g ${g}.bed >>& bed.log 
   end
   # Changed track type in trackDb/human/trackDb.ra to bed 12 and 
   # then did make alpha in trackDb directory.

   # trackDb/human/trackDb.ra entry was updated to include views for Raw Signal
   # and Alignment subtracks (2010-02-17)
   # 2010-05-15 and 2010-05-16. Add a Signal track so it is easier to view the data in 
   # regions where there is a high density of reads. 
   cd /hive/data/genomes/hg19/bed/burgeRnaSeqGemMapperAlign
   # Use bedItemOverlapCount to get counts of overlapping items for each base.
   # Need to sort the bed files and then get the number of reads mapped for
   # that tissue. Divide the counts by the number of million mapped reads to 
   # get the number of reads per million mapped reads as the data value. 
   # Re-make the subtracks using the -bed12 option so that blocks are used 
   # instead of just the first three fields of the BED file as is the default.
   rm *.count *.bedGraph
   foreach f (`ls *.bed`)
      echo $f
      set g=$f:r
      sort $f | bedItemOverlapCount -bed12 hg19 stdin > ${f}.count
      set size=`hgsql -Ne "select count(distinct name) from ${g};" hg19`
      awk -v size=${size} 'BEGIN {OFS="\t"} {print $1,$2,$3,($4 / (size/1000000));}' ${f}.count > ${g}.bedGraph
   end
   # Load the bedGraph tables into the database as Raw Signal tracks.
   foreach f (`ls *.bedGraph`)
      echo $f
      set g=$f:r
      hgsql -e "drop table ${g}AllRawSignal;" hg19 
      hgLoadBed -bedGraph=4 hg19 ${g}AllRawSignal $f >>& loadSignal.log
   end

  
##########################################################################
# BUILD ALLEN BRAIN TRACK (DONE 09/30/09 kent)

# Make the working directory
    ssh hgwdev
    cd /cluster/data/hg19/bed
    mkdir allenBrain
    cd allenBrain

# Remap the probe alignments from mm7 to hg19

    zcat /gbdb/mm9/liftOver/mm9ToHg19.over.chain.gz \
        |  pslMap -chainMapFile -swapMap \
	       /cluster/data/mm9/bed/allenBrain/allenBrainAli.psl stdin stdout \
	  |  sort -k 14,14 -k 16,16n > unscored.psl

    pslRecalcMatch unscored.psl /cluster/data/hg19/hg19.2bit \
        /cluster/data/mm9/bed/allenBrain/allenBrainProbes.fa allenBrainAli.psl

# Load the database
   hgsql hg19 < ~/kent/src/hg/lib/allenBrainUrl.sql
   hgsql hg19 -e 'load data local infile "/cluster/data/mm9/bed/allenBrain/allenBrainUrl.tab" into table allenBrainUrl;'
   hgLoadPsl hg19 allenBrainAli.psl
   mkdir /gbdb/hg19/allenBrain
   ln -s /cluster/data/mm9/bed/allenBrain/allenBrainProbes.fa /gbdb/hg19/allenBrain/allenBrainProbes.fa
   hgLoadSeq hg19 /gbdb/hg19/allenBrain/allenBrainProbes.fa

# Make mapping between known genes and allenBrain
   hgMapToGene hg19 allenBrainAli -type=psl knownGene knownToAllenBrain

#############################################################################
# ADD ALLEN BRAIN CORTEXT LINK (DONE, 11/18/09 kent)

# Copy over version from hg18 since we don't have new data from Allen Brain
# Inst.
    cd /cluster/data/hg19/bed/allenBrain
    cp /cluster/data/hg18/bed/allenBrain/allenBrainGene.tab .

# Load it into database.

    hgsql hg19 < ~/src/hg/lib/allenBrainGene.sql
    hgsql hg19 -e \
    'load data local infile "allenBrainGene.tab" into table allenBrainGene'


############################################################################
## Annotate 46-way multiple alignment with gene annotations
##		(DONE - 2008-12-08,23 - Hiram)
    # Gene frames
    ## survey all genomes to see what type of gene track to use
    ssh hgwdev
    mkdir /hive/data/genomes/hg19/bed/multiz46way/frames
    cd /hive/data/genomes/hg19/bed/multiz46way/frames
    #
    #	survey all the genomes to find out what kinds of gene tracks they have
    cat << '_EOF_' > showGenes.csh
#!/bin/csh -fe
foreach db (`cat ../species.list`)
    echo -n "${db}: "
    set tables = `hgsql $db -N -e "show tables like '%Gene%'"`
    foreach table ($tables)
	if ($table == "ensGene" || $table == "refGene" || $table == "mgcGenes" || \
	    $table == "knownGene" || $table == "xenoRefGene" ) then
		set count = `hgsql $db -N -e "select count(*) from $table"`
		echo -n "${table}: ${count}, "
	endif
    end
    set orgName = `hgsql hgcentraltest -N -e \
	    "select scientificName from dbDb where name='$db'"`
    set orgId = `hgsql hg19 -N -e \
	    "select id from organism where name='$orgName'"`
    if ($orgId == "") then
	echo "Mrnas: 0"
    else
	set count = `hgsql hg19 -N -e "select count(*) from gbCdnaInfo where organism=$orgId"`
	echo "Mrnas: ${count}"
    endif
end
'_EOF_'
    # << happy emacs
    chmod +x ./showGenes.csh
    #	rearrange that output to create four sections:
    #	1. knownGenes for hg19, mm9, rn4
    #	2. ensGene for almost everything else
    #	3. xenoRefGene for calJac1, petMar1, loxAfr3, papHam1, macEug1, oryCun2

    mkdir genes
    # knownGene
    for DB in hg19 mm9 rn4
do
    hgsql -N -e "select name,chrom,strand,txStart,txEnd,cdsStart,cdsEnd,exonCount,exonStarts,exonEnds from knownGene" ${DB} \
      | genePredSingleCover stdin stdout | gzip -2c \
        > /scratch/tmp/${DB}.tmp.gz
    mv /scratch/tmp/${DB}.tmp.gz genes/$DB.gp.gz
    echo "${DB} done"
done

    echo "panTro2 gorGor1 ponAbe2 rheMac2 tarSyr1 micMur1 otoGar1 \
	tupBel1 dipOrd1 cavPor3 speTri1 ochPri2 vicPac1 turTru1 \
	bosTau4 equCab2 felCat3 canFam2 myoLuc1 pteVam1 eriEur1 sorAra1 \
	proCap1 echTel1 dasNov2 choHof1 monDom5 ornAna1 galGal3 \
	taeGut1 anoCar1 xenTro2 tetNig2 fr2 gasAcu1 oryLat2 danRer6" \
    | sed -e "s/  */ /g" > ensGene.list


do
    # ensGene
    for DB in panTro2 gorGor1 ponAbe2 rheMac2 tarSyr1 micMur1 otoGar1 \
	tupBel1 dipOrd1 cavPor3 speTri1 ochPri2 vicPac1 turTru1 \
	bosTau4 equCab2 felCat3 canFam2 myoLuc1 pteVam1 eriEur1 sorAra1 \
	proCap1 echTel1 dasNov2 choHof1 monDom5 ornAna1 galGal3 \
	taeGut1 anoCar1 xenTro2 tetNig2 fr2 gasAcu1 oryLat2 danRer6
do
    hgsql -N -e "select name,chrom,strand,txStart,txEnd,cdsStart,cdsEnd,exonCount,exonStarts,exonEnds from ensGene" ${DB} \
      | genePredSingleCover stdin stdout | gzip -2c \
        > /scratch/tmp/${DB}.tmp.gz
    mv /scratch/tmp/${DB}.tmp.gz genes/$DB.gp.gz
    echo "${DB} done"
done

    echo "calJac1 petMar1 loxAfr3 papHam1 macEug1 oryCun2" > xenoRef.list

    # xenoRefGene
    for DB in calJac1 petMar1 loxAfr3 papHam1 macEug1 oryCun2
do
    hgsql -N -e "select name,chrom,strand,txStart,txEnd,cdsStart,cdsEnd,exonCount,exonStarts,exonEnds from xenoRefGene" ${DB} \
      | genePredSingleCover stdin stdout | gzip -2c \
        > /scratch/tmp/${DB}.tmp.gz
    mv /scratch/tmp/${DB}.tmp.gz genes/$DB.gp.gz
    echo "${DB} done"
done

    #	the following single command doesn't work on any 32 Gb computer,
    #	requires much more memory, turn it into a kluster job, see below ...

    #	Create this command with this script:
    cat << '_EOF_' > mkCmd.sh
#!/bin/sh

echo "time (cat ../maf/*.maf | nice -n +19 genePredToMafFrames hg19 stdin stdout \\"
for G in mm9 rn4
do
    if [ ! -s genes/${G}.gp.gz ]; then
	echo "missing genes/${G}.gp.gz"
	exit 255
    fi
    echo -n "${G} genes/${G}.gp.gz "
done
echo "\\"
for D in `sort ensGene.list`
do
    if [ ! -s genes/${D}.gp.gz ]; then
        echo "missing genes/${D}.gp.gz"
        exit 255
    fi
    echo -n "${D} genes/${D}.gp.gz "
done
echo "\\"
for D in `sort xenoRef.list`
do
    if [ ! -s genes/${D}.gp.gz ]; then
        echo "missing genes/${D}.gp.gz"
        exit 255
    fi
    echo -n "${D} genes/${D}.gp.gz "
done
echo "\\"
echo "    | gzip > multiz46way.mafFrames.gz) > frames.log 2>&1"
'_EOF_'
    # << happy emacs
    chmod +x ./mkCmd.sh

    #	this doesn't work on any 32 Gb computer, requires much more memory
    #	turn it into a kluster job, see below
    time (cat ../maf/*.maf | nice -n +19 genePredToMafFrames hg19 stdin stdout \
mm9 genes/mm9.gp.gz rn4 genes/rn4.gp.gz \
panTro2 genes/panTro2.gp.gz gorGor1 genes/gorGor1.gp.gz ponAbe2 genes/ponAbe2.gp.gz rheMac2 genes/rheMac2.gp.gz tarSyr1 genes/tarSyr1.gp.gz micMur1 genes/micMur1.gp.gz otoGar1 genes/otoGar1.gp.gz tupBel1 genes/tupBel1.gp.gz dipOrd1 genes/dipOrd1.gp.gz cavPor3 genes/cavPor3.gp.gz speTri1 genes/speTri1.gp.gz ochPri2 genes/ochPri2.gp.gz vicPac1 genes/vicPac1.gp.gz turTru1 genes/turTru1.gp.gz bosTau4 genes/bosTau4.gp.gz equCab2 genes/equCab2.gp.gz felCat3 genes/felCat3.gp.gz canFam2 genes/canFam2.gp.gz myoLuc1 genes/myoLuc1.gp.gz pteVam1 genes/pteVam1.gp.gz eriEur1 genes/eriEur1.gp.gz sorAra1 genes/sorAra1.gp.gz proCap1 genes/proCap1.gp.gz echTel1 genes/echTel1.gp.gz dasNov2 genes/dasNov2.gp.gz choHof1 genes/choHof1.gp.gz monDom5 genes/monDom5.gp.gz ornAna1 genes/ornAna1.gp.gz galGal3 genes/galGal3.gp.gz taeGut1 genes/taeGut1.gp.gz anoCar1 genes/anoCar1.gp.gz xenTro2 genes/xenTro2.gp.gz tetNig2 genes/tetNig2.gp.gz fr2 genes/fr2.gp.gz gasAcu1 genes/gasAcu1.gp.gz oryLat2 genes/oryLat2.gp.gz danRer6 genes/danRer6.gp.gz \
calJac1 genes/calJac1.gp.gz petMar1 genes/petMar1.gp.gz loxAfr3 genes/loxAfr3.gp.gz papHam1 genes/papHam1.gp.gz macEug1 genes/macEug1.gp.gz oryCun2 genes/oryCun2.gp.gz \
    | gzip > multiz46way.mafFrames.gz) > frames.log 2>&1

    #	that doesn't work on any 32 Gb computer, requires much more memory
    #	turn it into a kluster job
    ssh swarm
    cd /hive/data/genomes/hg19/bed/multiz46way/frames
    cat << '_EOF_' > runOne
#!/bin/csh -fe

set C = $1
set G = $2

cat ../maf/${C}.maf | genePredToMafFrames hg19 stdin stdout \
        ${G} genes/${G}.gp.gz | gzip > parts/${C}.${G}.mafFrames.gz
'_EOF_'
    # << happy emacs
    chmod +x runOne

    ls ../maf | sed -e "s/.maf//" > chr.list
    ls genes | sed -e "s/.gp.gz//" | grep -v hg19 > gene.list

    cat << '_EOF_' > template
#LOOP
runOne $(root1) $(root2) {check out exists+ parts/$(root1).$(root2).mafFrames.gz}
#ENDLOOP
'_EOF_'
    # << happy emacs

    mkdir parts
    gensub2 chr.list gene.list template jobList
    para -ram=8g create jobList
    para try ... check ... push
# Completed: 4185 of 4185 jobs
# CPU time in finished jobs:      72491s    1208.19m    20.14h    0.84d  0.002 y
# IO & Wait Time:               1462162s   24369.36m   406.16h   16.92d  0.046 y
# Average job time:                 367s       6.11m     0.10h    0.00d
# Longest finished job:            3165s      52.75m     0.88h    0.04d
# Submission to last job:          6364s     106.07m     1.77h    0.07d

    # see what it looks like in terms of number of annotations per DB:
    find ./parts -type f | while read F
do
    zcat ${F}
done | cut -f4 | sort | uniq -c | sort -n > annotation.survey.txt
  79191 rn4
 108287 petMar1
 139581 gorGor1
 140487 taeGut1
 143058 choHof1
 143233 vicPac1
 150073 anoCar1
 154462 tarSyr1
 163930 sorAra1
 164575 galGal3
 171191 macEug1
 174221 felCat3
 175831 dasNov2
 177622 ornAna1
 190729 eriEur1
 192285 tupBel1
 198052 speTri1
 199639 micMur1
 201731 papHam1
 201961 panTro2
 206170 oryCun2
 209327 ponAbe2
 209504 otoGar1
 210860 rheMac2
 212533 proCap1
 212848 myoLuc1
 213146 dipOrd1
 213479 calJac1
 215995 echTel1
 220341 ochPri2
 225132 loxAfr3
 226689 turTru1
 230903 monDom5
 232025 pteVam1
 232831 equCab2
 236945 cavPor3
 238167 bosTau4
 239857 mm9
 255727 canFam2
 316850 xenTro2
 359507 danRer6
 375156 oryLat2
 390076 fr2
 426532 gasAcu1
 434619 tetNig2

    #	load the resulting file
    ssh hgwdev
    cd /cluster/data/hg19/bed/multiz46way/frames
    find ./parts -type f | while read F
do
    zcat ${F}
done | sort -k1,1 -k2,2n > multiz46wayFrames.bed

    hgLoadMafFrames hg19 multiz46wayFrames stdin

    featureBits -countGaps hg19 multiz46wayFrames.bed
    #	57146632 bases of 3137161264 (1.822%) in intersection

    #	enable the trackDb entries:
# frames multiz46wayFrames
# irows on
    #	appears to work OK

#############################################################################
## create upstream refGene maf files
    cd /hive/data/genomes/hg19/bed/multiz46way/downloads/maf
    # bash script
#!/bin/sh
for S in 1000 2000 5000
do
    echo "making upstream${S}.maf"
    featureBits hg19 refGene:upstream:${S} -fa=/dev/null -bed=stdout \
        | perl -wpe 's/_up[^\t]+/\t0/' | sort -k1,1 -k2,2n \
        | /cluster/bin/$MACHTYPE/mafFrags hg19 multiz46way \
                stdin stdout \
                -orgs=/hive/data/genomes/hg19/bed/multiz46way/species.list \
        | gzip -c > upstream${S}.maf.gz
    echo "done upstream${S}.maf.gz"
done

    cd /usr/local/apache/htdocs/goldenPath/hg19/multiz46way/maf
    ln -s /hive/data/genomes/hg19/bed/multiz46way/downloads/maf/up*.gz .
    md5sum up*.gz >> md5sum.txt


#############################################################################
# AFFY U133AB (Done - 2009-09-30 - Jim)
    # Align probes 
    ssh swarm
    cd /cluster/data/hg19/bed
    mkdir -p affyProbes/affyU133/run
    cd affyProbes/affyU133/run
    mkdir psl
    ls -1 /scratch/data/hg19/nib/*.nib > genome.lst
    ls -1 /hive/data/outside/affyProbes/HG-U133AB_all.fa > mrna.lst

    cat << '_EOF_' > gsub
#LOOP
/cluster/bin/x86_64/blat -fine -ooc=/scratch/data/hg19/11.ooc  $(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    gensub2 genome.lst mrna.lst gsub jobList
    para create jobList
    para try
    para check
    para push
    para time
#Completed: 93 of 93 jobs
#CPU time in finished jobs:      21246s     354.09m     5.90h    0.25d  0.001 y
#IO & Wait Time:                   349s       5.82m     0.10h    0.00d  0.000 y
#Average job time:                 232s       3.87m     0.06h    0.00d
#Longest finished job:            1650s      27.50m     0.46h    0.02d
#Submission to last job:          1685s      28.08m     0.47h    0.02d


    # Do sort, best in genome filter.
    # to create affyU133.psl.
    pslSort dirs raw.psl tmp psl
    pslReps -minCover=0.3 -minAli=0.95 -nearTop=0.005 raw.psl ../affyU133.psl /dev/null
    rm -r raw.psl psl

    # Load probes and alignments into database.
    ssh hgwdev
    cd /cluster/data/hg19/bed/affyProbes/affyU133
    hgLoadPsl hg19 affyU133.psl
    hgLoadSeq hg19 /gbdb/hgFixed/affyProbes/HG-U133AB_all.fa

    # Added knownToU133 table
    hgMapToGene hg19 affyU133 knownGene knownToU133
    # trim unwanted chip-prefix to be backwards compatible with hg17 and hg18
    hgsql hg19 -e 'update knownToU133 set value=substring(value,7)'

    # remove the trailing ";" from the value field (redmine #1685)
    hgsql hg19 -e 'update knownToU133 set value=trim(trailing ";" from value);'

##########################################################################
# GNF ATLAS 2 (Done - 2009-09-30 - Jim)
    # Align probes from GNF1H chip.
    ssh swarm
    cd /cluster/data/hg19/bed
    mkdir -p geneAtlas2/run/psl
    cd geneAtlas2/run
    mkdir psl
    ls -1 /scratch/data/hg19/nib/*.nib > genome.lst
    ls -1 /hive/data/outside/gnf/human/atlas2/gnf1h.fa > mrna.lst
    cat << '_EOF_' > gsub
#LOOP
/cluster/bin/x86_64/blat -fine -ooc=/scratch/data/hg19/11.ooc  $(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    gensub2 genome.lst mrna.lst gsub jobList
    para create jobList
    para try
    para check
    para push
    para time
#Completed: 93 of 93 jobs
#CPU time in finished jobs:       3299s      54.98m     0.92h    0.04d  0.000 y
#IO & Wait Time:                   330s       5.50m     0.09h    0.00d  0.000 y
#Average job time:                  39s       0.65m     0.01h    0.00d
#Longest finished job:             370s       6.17m     0.10h    0.00d
#Submission to last job:           477s       7.95m     0.13h    0.01d


    # Do sort, best in genome filter
    # to create gnf1h.psl.
    pslSort dirs raw.psl tmp psl
    pslReps -minCover=0.3 -minAli=0.95 -nearTop=0.005 raw.psl ../affyGnf1h.psl /dev/null
    rm -r raw.psl psl

    # Load probes and alignments from GNF1H into database.
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/geneAtlas2
    hgLoadPsl hg19 affyGnf1h.psl
    hgLoadSeq hg19 /gbdb/hgFixed/affyProbes/gnf1h.fa

    grep -v U133B ../affyProbes/affyU133/affyU133.psl \
	| sed -e "s/exemplar://; s/consensus://; s/U133A://" \
	| sed -e "s/;//" > affyU133A.psl

    hgMapMicroarray gnfAtlas2.bed hgFixed.gnfHumanAtlas2MedianRatio \
    	affyU133A.psl  affyGnf1h.psl

    # Loaded 44696 rows of expression data from hgFixed.gnfHumanAtlas2MedianRatio
    # Mapped 33186,  multiply-mapped 3171, missed 48, unmapped 11510

    hgLoadBed hg19 gnfAtlas2 gnfAtlas2.bed
    # Loaded 36357 elements of size 15

    # Added knownToGnf1h table
    hgMapToGene hg19 affyGnf1h knownGene knownToGnf1h

##########################################################################
# BUILD NIBB IMAGE PROBES (DONE 2009-10-12 JK)

# Make directory on san for cluster job and copy in sequence
    ssh swarm
    mkdir /hive/data/genomes/hg19/bed/nibbPics
    cd /hive/data/genomes/hg19/bed/nibbPics
    cp /cluster/data/xenTro1/bed/nibbPics/nibbImageProbes.fa .

# Make parasol job dir and sequence list files
    mkdir run
    cd run
    mkdir psl
    ls -1 /scratch/data/hg19/nib/*.nib > genome.lst
    echo ../nibbImageProbes.fa > mrna.lst

# Create parasol gensub file file
cat << '_EOF_' > gsub
#LOOP
blatz -rna -minScore=6000 -out=psl $(path1) $(path2) psl/$(root1)_$(root2).psl
#ENDLOOP
'_EOF_'
# << emacs

# Create parasol batch
    gensub2 genome.lst mrna.lst gsub spec
    para create spec

# Do para try/push/time etc.
#Completed: 93 of 93 jobs
#CPU time in finished jobs:       8008s     133.47m     2.22h    0.09d  0.000 y
#IO & Wait Time:                   364s       6.07m     0.10h    0.00d  0.000 y
#Average job time:                  90s       1.50m     0.03h    0.00d
#Longest finished job:             765s      12.75m     0.21h    0.01d
#Submission to last job:           824s      13.73m     0.23h    0.01d

# Make sort and filter
    catDir psl | sort -k 10 \
        | pslReps stdin stdout /dev/null -nohead -minAli=0.60 -nearTop=0.001 -minCover=0.10 -minNearTopSize=80 \
	| sort -k 14,14 -k 16,16n \
	| sed 's#/scratch/data/hg19/nib/chr#chr#' \
	| sed 's/.nib//' > ../nibbImageProbes.psl

# Make bed file and copy in stuff
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/nibbPics
    cp /cluster/data/xenTro1/bed/nibbPics/nibbImageProbes.fa .

# Load into database
    ln -s /cluster/data/hg19/bed/nibbPics/nibbImageProbes.fa /gbdb/hg19/nibbImageProbes.fa
    hgLoadSeq hg19 /gbdb/hg19/nibbImageProbes.fa
    hgLoadPsl hg19 nibbImageProbes.psl

##########################################################################
# Initial vgProbeTrack run for hg19 (galt 2009-10-15)
# see visiGene.txt make doc
# uses nibbImageProbes and vgProbeTrack utility
# creates vgAllProbes and knownToVisiGene
#    25931 
# updates visiGene.vgPrbAliAll.
# creates and runs hgLoadSeq on /gbdb/hg19/visiGene/*.fa

##########################################################################
# make new grp table to match hg18 (DONE  2009-10-01 kuhn)
# to split regulation from expression 
# phenDis group is also missing in hg19
# and add one more column: defaultIsClosed

# get the hg18.grp table into hg19 

# copy the hg18.grp table into hg19.grpNew and edit 
   hgsql hg19
   CREATE TABLE grpNew SELECT * FROM hg18.grp;
   # 24 rows in set (0.00 sec)
   
   DELETE FROM grpNew WHERE name LIKE "encode%";
   DELETE FROM grpNew WHERE name LIKE "remc%"; 
   DELETE FROM grpNew WHERE name LIKE "tcga%";
   DELETE FROM grpNew WHERE name LIKE "cancer%"; 
   DELETE FROM grpNew WHERE name LIKE "jk%";
   # 10 rows in set (0.00 sec)
   
# move the new table into place quickly
   DROP TABLE grp;
   RENAME TABLE grpNew TO grp;
 
#########################################################################
# BUILD OMIM RELATED GENES TRACK (done 2009-10-13 jk)

ssh hgwdev
cd /hive/data/genomes/hg19/bed
mkdir omimGene
cd omimGene

# download the file morbidmap and genemap from OMIM

mkdir omim
cd omim
wget --timestamping ftp://ftp.ncbi.nih.gov/repository/OMIM/morbidmap
wget --timestamping ftp://ftp.ncbi.nih.gov/repository/OMIM/genemap
cat genemap|sed -e 's/|/\t/g' > genemap.tab
autoSql ~/src/hg/lib/omimGeneMap.as x
cat x.sql |sed -e 's/PRIMARY KEY(numbering)/KEY(omimId)/' >omimGeneMap.sql
hgLoadSqlTab -warn hg19 omimGeneMap omimGeneMap.sql genemap.tab

# got warning on 3 records, just ignore them
# Warning: load of omimGeneMap did not go as planned: 12216 record(s), 0 row(s)

rm x.c x.h
cd ..
cat omim/morbidmap|sed -e 's/|/\t/g' > mobidmap.tab
autoSql ~/src/hg/lib/omimMorbidMap.as x 
cat x.sql |sed -e 's/PRIMARY KEY(description)/KEY(omimId)/' >omimMorbidMap.sql
hgLoadSqlTab -warn hg19 omimMorbidMap omimMorbidMap.sql mobidmap.tab

# get all UCSC genes (from the knownGene table) that cross-reference to a RefSeq gene 
# that has a non-empty OMIM ID according to the refLink table.  And use OMIM ID as
# the gene name for this new table.  Please note the alignId field still holds the KG ID.

hgsql hg19 -N -e \
'select omimId, kg.* from knownGene kg, knownToRefSeq kr, refLink l where omimId != 0 and mrnaAcc=kr.value and kg.name=kr.name ' \
|cut -f 1,3-13 >o1.tab

# collect more OMIM related genes via the MIM external DB links from UniProt

hgsql hg19 -N -e \
'select extAC, kg.* from knownGene kg, kgXref k, proteome.spXref2 p where spId=p.accession and extDB="MIM" and kg.name=kgId ' \
|cut -f 1,3-13 >o2.tab

# concatenate the above two gene sets and remove duplications.

cat o1.tab o2.tab |sort -u >o3.tab

# load the result into a temp table, fanO3
hgLoadSqlTab hg19 fanO3 ~/src/hg/lib/knownGene.sql o3.tab

# while holding onto the OMIM ID, get the canonical gene (via the knownGene, knowIsoforms, 
# and knownCanonical tables) that represent a cluster which contains 
# initial OMIM gene in the fanO3 table

hgsql hg19 -N -e \
'select f3.name, kg.* from fanO3 f3, knownGene kg, knownCanonical c, knownIsoforms i where f3.alignId=i.transcript and kg.name=c.transcript and c.clusterId=i.clusterId'\
> o4.tab

# first column is the OMIM ID
cut -f 1 o4.tab >j1.tmp

# col 3-13 is the gene structure of the canonical KG
cut -f 3-13 o4.tab >j2.tmp

# stitch them together and remove duplicates, load the result into fanO4 table
paste j1.tmp j2.tmp |sort -u >fanO4.tab
hgLoadSqlTab hg19 fanO4  ~/src/hg/lib/knownGene.sql fanO4.tab

# finally sort the table and create bed 4 file and load it as the omimGene table

hgsql hg19 -N -e 'select chrom, txStart, txEnd, name from fanO4 order by chrom, txStart, txEnd' |sort -u >omimGene.bed
hgLoadBed hg19 omimGene omimGene.bed

# create and load the omimToKnownCanonical table.

hgsql hg19 -N -e 'select name, alignId from fanO4 order by name'\
> omimToKnownCanonical.tab

hgLoadSqlTab hg19 omimToKnownCanonical  \
~/src/hg/lib/omimToKnownCanonical.sql omimToKnownCanonical.tab

# The following clean up could be done.
# hgsql hg19 -e 'drop table fanO3'
# hgsql hg19 -e 'drop table fanO4'
# rm j*.tmp
# rm o1.tab o2.tab o3.tab o4.tab

#########################################################################
# BUILD HPRD DATA FOR KNOWN GENE DETAILS PAGE LINKS (in progress 2009-10-14 jk)

# Make the directory to work in
   cd /hive/data/genomes/hg19/bed
   mkdir hprd
   cd hprd

# Download HPRD_XML_070609.tar.gz from www.hprd.org. Unfortunately this
# requires registration, so can't just wget it.

    zcat HPRD_XML_070609.tar.gz | tar -xv

# This will create 20000 or more  xxxx.xml files under HPRD_XML_070609

# Create hprdToCdna table
    echo HPRD_XML_070609/*.xml | xargs grep entry_cdna  > j.cdna
    cat j.cdna| sed -e 's/\//\t/' | sed -e 's/.xml/\t/' |\
	sed -e 's/<entry_cdna>/\t/' | sed -e 's/<\//\t/'| sed -e 's/\./\t/'| cut -f 2,4|\
	grep -v None >hprdToCdna.tab

    hgsql hg19 <~/src/hg/lib/hprdToCdna.sql
    hgsql hg19 -e 'load data local infile "hprdToCdna.tab" into table hprdToCdna'

# Create hprdToUniProt table

    echo 'fgrep -H Swiss  HPRD_XML_070609/$1.xml' >do1

    ls HPRD_XML_070609 >j
    cat j |sed -e 's/.xml/\tdo1/g' >jj
    cut -f 1 jj >j.2
    cut -f 2 jj >j.1
    paste j.1 j.2 >doall
    chmod +x do*

    ./doall >j.out
    cat j.out|grep SwissProt | sed -e 's/\//\t/' | sed -e 's/.xml/\t/' | \
    sed -e 's/Prot>/\t/' | sed -e 's/<\//\t/'| cut -f 2,4|grep -v None >hprdToUniProt.tab

    hgsql hg19 <~/src/hg/lib/hprdToUniProt.sql
    hgsql hg19 -e 'load data local infile "hprdToUniProt.tab" into table hprdToUniProt'

# build knownToHprd table

    hgsql hg19 -N -e 'select kgId,hprdId from hprdToCdna, kgXref where cdnaId=refseq' >j.kg1
    hgsql hg19 -N -e 'select kgId,hprdId from hprdToUniProt, kgXref where uniProtId=spId' >j.kg2

    cat j.kg1 j.kg2 | sed 's/_.//' | sort -u >knownToHprd.tab
    wc knownToHprd.tab

    hgsql hg19 <~/src/hg/lib/knownToHprd.sql

    hgsql hg19 -e 'load data local infile "knownToHprd.tab" into table knownToHprd'
    hgsql hg19 -e 'select count(*) from knownToHprd'

# 21,516 records created

# remove temporary files.

    rm j*

#########################################################################
# hgPal downloads (DONE braney 2009-11-03)
#   FASTA from 46way for refGene, knownGene, knownCanonical 

    ssh hgwdev
    screen
    bash
    rm -rf /cluster/data/hg19/bed/multiz46way/pal
    mkdir /cluster/data/hg19/bed/multiz46way/pal
    cd /cluster/data/hg19/bed/multiz46way/pal
    for i in `cat ../species.list`; do echo $i; done > order.lst

    mz=multiz46way
    gp=refGene
    db=hg19
    mkdir exonAA exonNuc ppredAA ppredNuc
    for j in `sort -nk 2 /cluster/data/$db/chrom.sizes | awk '{print $1}'`
    do
	echo "date"
	echo "mafGene -chrom=$j  $db $mz $gp order.lst stdout | \
	    gzip -c > ppredAA/$j.ppredAA.fa.gz"
	echo "mafGene -chrom=$j -noTrans $db $mz $gp order.lst stdout | \
	    gzip -c > ppredNuc/$j.ppredNuc.fa.gz"
	echo "mafGene -chrom=$j -exons -noTrans $db $mz $gp order.lst stdout | \
	    gzip -c > exonNuc/$j.exonNuc.fa.gz"
	echo "mafGene -chrom=$j -exons $db $mz $gp order.lst stdout | \
	    gzip -c > exonAA/$j.exonAA.fa.gz"
    done > $gp.jobs

    time sh -x $gp.jobs > $gp.jobs.log 2>&1 &
    sleep 1
    tail -f $gp.jobs.log

# real    350m10.116s
# user    38m38.731s
# sys     7m22.106s


    mz=multiz46way
    gp=refGene
    db=hg19
    zcat exonAA/*.gz | gzip -c > $gp.$mz.exonAA.fa.gz
    zcat exonNuc/*.gz | gzip -c > $gp.$mz.exonNuc.fa.gz
    zcat ppredAA/*.gz | gzip -c > $gp.$mz.ppredAA.fa.gz
    zcat ppredNuc/*.gz | gzip -c > $gp.$mz.ppredNuc.fa.gz

    rm -rf exonAA exonNuc ppredAA ppredNuc

    # we're only distributing exons at the moment
    mz=multiz46way
    gp=refGene
    db=hg19
    pd=/usr/local/apache/htdocs/goldenPath/$db/$mz/alignments
    mkdir -p $pd
    ln -s `pwd`/$gp.$mz.exonAA.fa.gz $pd/$gp.exonAA.fa.gz
    ln -s `pwd`/$gp.$mz.exonNuc.fa.gz $pd/$gp.exonNuc.fa.gz

    mz=multiz46way
    gp=knownGene
    db=hg19
    mkdir exonAA exonNuc ppredAA ppredNuc
    for j in `sort -nk 2 /cluster/data/$db/chrom.sizes | awk '{print $1}'`
    do
	echo "date"
	echo "mafGene -chrom=$j  $db $mz $gp order.lst stdout | \
	    gzip -c > ppredAA/$j.ppredAA.fa.gz"
	echo "mafGene -chrom=$j -noTrans $db $mz $gp order.lst stdout | \
	    gzip -c > ppredNuc/$j.ppredNuc.fa.gz"
	echo "mafGene -chrom=$j -exons -noTrans $db $mz $gp order.lst stdout | \
	    gzip -c > exonNuc/$j.exonNuc.fa.gz"
	echo "mafGene -chrom=$j -exons $db $mz $gp order.lst stdout | \
	    gzip -c > exonAA/$j.exonAA.fa.gz"
    done > $gp.$mz.jobs

    time sh -x $gp.$mz.jobs > $gp.$mz.job.log 2>&1 &
    sleep 1
    tail -f $gp.$mz.job.log

# oops... missed the timing


    mz=multiz46way
    gp=knownGene
    db=hg19

    zcat exonAA/c*.gz | gzip -c > $gp.$mz.exonAA.fa.gz
    zcat exonNuc/c*.gz | gzip -c > $gp.$mz.exonNuc.fa.gz
    zcat ppredAA/c*.gz | gzip -c > $gp.$mz.ppredAA.fa.gz
    zcat ppredNuc/c*.gz | gzip -c > $gp.$mz.ppredNuc.fa.gz

    rm -rf exonAA exonNuc ppredAA ppredNuc

    mz=multiz46way
    gp=knownGene
    db=hg19
    pd=/usr/local/apache/htdocs/goldenPath/$db/$mz/alignments
    mkdir -p $pd
    ln -s `pwd`/$gp.$mz.exonAA.fa.gz $pd/$gp.exonAA.fa.gz
    ln -s `pwd`/$gp.$mz.exonNuc.fa.gz $pd/$gp.exonNuc.fa.gz

    # now do the canonical set
    cd /cluster/data/hg19/bed/multiz46way/pal
    mz=multiz46way
    gp=knownCanonical
    db=hg19
    for j in `awk '{print $1}' /cluster/data/hg19/chrom.sizes`
    do
	echo "select chrom, chromStart, chromEnd, transcript from knownCanonical where chrom='$j'" | hgsql $db | tail -n +2 > $j.known.bed
    done

    mkdir exonAA exonNuc ppredAA ppredNuc
    for j in `sort -nk 2 /cluster/data/$db/chrom.sizes | awk '{print $1}'`
    do
	echo "date"
	echo "mafGene -geneBeds=$j.known.bed  $db $mz knownGene order.lst stdout | \
	    gzip -c > ppredAA/$j.ppredAA.fa.gz"
	echo "mafGene -geneBeds=$j.known.bed -noTrans $db $mz knownGene order.lst stdout | \
	    gzip -c > ppredNuc/$j.ppredNuc.fa.gz"
	echo "mafGene -geneBeds=$j.known.bed -exons -noTrans $db $mz knownGene order.lst stdout | \
	    gzip -c > exonNuc/$j.exonNuc.fa.gz"
	echo "mafGene -geneBeds=$j.known.bed -exons $db $mz knownGene order.lst stdout | \
	    gzip -c > exonAA/$j.exonAA.fa.gz"
    done > $gp.$mz.jobs

    time sh -x $gp.$mz.jobs > $gp.$mz.job.log 2>&1 &
    sleep 1
    tail -f $gp.$mz.job.log

# real    302m20.489s
# user    27m31.179s
# sys     5m30.071s


    rm *.known.bed
    mz=multiz46way
    gp=knownCanonical
    db=hg19
    zcat exonAA/c*.gz | gzip -c > $gp.$mz.exonAA.fa.gz
    zcat exonNuc/c*.gz | gzip -c > $gp.$mz.exonNuc.fa.gz
    zcat ppredAA/c*.gz | gzip -c > $gp.$mz.ppredAA.fa.gz
    zcat ppredNuc/c*.gz | gzip -c > $gp.$mz.ppredNuc.fa.gz

    rm -rf exonAA exonNuc ppredAA ppredNuc

    mz=multiz46way
    gp=knownCanonical
    db=hg19
    pd=/usr/local/apache/htdocs/goldenPath/$db/$mz/alignments
    mkdir -p $pd
    ln -s `pwd`/$gp.$mz.exonAA.fa.gz $pd/$gp.exonAA.fa.gz
    ln -s `pwd`/$gp.$mz.exonNuc.fa.gz $pd/$gp.exonNuc.fa.gz

############################################################################
# SEGMENTAL DUPLICATIONS (2010-02-02 - 2010-02-04, hartera, DONE)
# (2011-09-26, Fan, REBUILT using corrected data from provider)
# File emailed from Tin Louie <tinlouie at u.washington.edu>
# in Evan Eichler's lab on 01/28/10. This is a data update since it was
# thought that the last data set was incorrect so the pipeline had to be
# re-run.
# NOTE: Received e-mail from Tin Louie suggesting that the otherSize 
# column could be dropped. It is just the size of the otherChrom and it 
# does not seem to be used for the track display or details page. It has the
# correct description in the table schema so it is ok to keep it for now. 
# In the future, this column could be dropped if it not useful. 
# There are a number of columns that could be dropped as they are 
# meaningless but decided to keep them as the code for the details page
# expect them to be there.
# 01/28/10 Received new data as previous run of the pipeline may have
# produced incorrect results. 
# 2010-02-02 Loader aborted on data since in some lines there was an empty
# field so the loader read only 28 words instead of 29. E-mailed Tin to
# ask for the data to be fixed. 
# 2010-02-03 Received new data as the previous data had empty fields.
# 2010-02-04 Loaded new data into hg19 database.
# 2010-02-09 Received new data on 02/08/10 as there were more errors in the
# code that caused the data to have empty fields.
# 2010-02-19 Changed the posBasesHit column values to match those for hg18.
# 2011-09-26 Rebuilt, using corrected (10th col) from data provider.
# In hg18, they are all 1000, but this is meaningless. 
    mkdir /hive/data/genomes/hg19/bed/genomicSuperDups/0926_2011
    cd /hive/data/genomes/hg19/bed/genomicSuperDups/0926_2011
    wget --timestamping \
         ftp://mesh.gs.washington.edu/pub/UCSC/hg19genomicSuperDups.fixed.tab.gz
    gunzip hg19genomicSuperDups.fixed.tab.gz
    # Fix incorrect chromosome names in data. Check both chrom and otherChrom.
    # Previously, found several cases where the last letter of random was
    # missing for the names of the random contigs. They all look good this
    # time.
    awk '{print $1}' hg19genomicSuperDups.fixed.tab | sort | uniq > chroms
    awk '{print $7}' hg19genomicSuperDups.fixed.tab | sort | uniq > otherChroms
    hgsql -Ne 'select chrom from chromInfo;' hg19 | sort | uniq > chromInfo.txt 
    comm -23 chroms chromInfo.txt
    comm -23 otherChroms chromInfo.txt
    # chroms and otherChroms match chromosome names in chromInfo.

    # The sed command is necessary to fix "_" used as strand to "-".
    # The awk command was necessary for some recent other species
    # genomicSuperDups that had some too-short regions.  It does not seem
    # to be necessary here, but doesn't hurt and may be useful in
    # future builds.
    hgsql -e 'drop table genomicSuperDups;' hg19
    sed -e 's/\t_\t/\t-\t/' hg19genomicSuperDups.fixed.tab \
    | awk '($3 - $2) >= 1000 && ($9 - $8) >= 1000 {print;}' \
    | hgLoadBed hg19 genomicSuperDups stdin \
      -sqlTable=$HOME/kent/src/hg/lib/genomicSuperDups.sql
# Loaded 51599 elements of size 29
# Sorted
# Creating table definition for genomicSuperDups
# Saving bed.tab
# Loading hg19

    # 2009-11-05: 
    # Updated details page with suggested text and an additional reference. 
    # src/hg/makeDb/trackDb/genomicSuperDups.html
    # 2010-02-04: Updated the schema description as below in
    # src/hg/lib/genomicSuperDups.sql. Kept score as it is used in older
    # datasets e.g. on hg18 -  
    # Suggestions by Tin Louie for the schema description:
# I suggest that the description of those meaningless columns (on the webpage
# 'Schema for Segmental Dups') be changed to "for future use". The meaningless
# columns are:  score, posBasesHit, testResult, verdict, chits, ccov
# The descriptions of other columns should be changed for clarification:
# otherSize -- equal to otherEnd minus otherStart
# uid -- id shared by the query & subject of a hit 
     # 2010-02-19 Changed the posBasesHit column to be 1000. Checked with 
     # data provider about doing this so that the values are the same as for
     # those in the hg18 table.
     # hgsql -e 'update genomicSuperDups set posBasesHit = 1000;' hg19
# New corrected data fixed the above problem. 
############################################################################
# ADD LINK TO GENENETWORK (DONE. 12/02/09 Fan).

# Received geneNetwork ID list file, GN_human_RefSeq.txt, for hg19 from GeneNetwork, Zhou Xiaodong [xiaodong.zhou@gmail.com].

    ssh hgwdev
    mkdir -p /cluster/data/hg19/bed/geneNetwork
    cd /cluster/data/hg19/bed/geneNetwork

    hgsql hg19 < ~/src/hg/lib/geneNetworkId.sql
    hgsql hg19 -e \
    'load data local infile "GN_human_RefSeq.txt" into table geneNetworkId'

######################################################################## 
# EXONIPHY (2009-12-28, hartera, DONE)
# 2010-01-05, hartera. Moved trackDb entry for exoniphy up to human directory
# level as it is the same for all assemblies.
    # New predictions for hg19 run by Melissa Hubisz 
    # (mjhubisz at gmail.com) for hg19 and sent by 
    # Adam Siepel (acs4 at cornell.edu) on 2009-12-18
    mkdir -p /hive/data/genomes/hg19/bed/exoniphy.2009-12-18
    cd /hive/data/genomes/hg19/bed/exoniphy.2009-12-18
    # Download predictions file, exoniphy.gff
    wget --timestamping \
         "http://compgen.bscb.cornell.edu/~acs/exoniphy.gff.gz"
    gunzip exoniphy.gff.gz
    # Remove table of lifted predictions from hg18
    hgsql -e 'drop table exoniphy;' hg19
    ldHgGene -genePredExt -gtf hg19 exoniphy exoniphy.gff
    # Read 620689 transcripts in 647299 lines in 1 files
    # 620689 groups 24 seqs 1 sources 4 feature types
    # 186601 gene predictions
    # Added a copy of the hg18 track description to trackDb/human/hg19 and 
    # updated it and added a trackDb entry to the trackDb.ra there.
    # 2010-01-04. Moved exoniphy trackDb entry up to the human level
    # trackDb.ra since the entry is the same for hg16-19. Removed the entry in
    # trackDb.ra in each of those assembly directories.

######################################################################## 
# Vega gene update (DONE - 2010-01-15 - Hiram)
    #	lookup version number at the Vega WEB site:
    #	http://vega.sanger.ac.uk/index.html
    #	and FTP site:
    #	ftp://ftp.sanger.ac.uk/pub/vega/
    cd /hive/data/genomes/hg19
    #	step wise to verify operation
    doEnsGeneUpdate.pl -vegaGene -ensVersion=36 -stop=download hg19.ensGene.ra
    doEnsGeneUpdate.pl -vegaGene -ensVersion=36 \
	-continue=process -stop=process hg19.ensGene.ra
    doEnsGeneUpdate.pl -vegaGene -ensVersion=36 \
	-continue=load -stop=load hg19.ensGene.ra
    doEnsGeneUpdate.pl -vegaGene -ensVersion=36 \
	-continue=cleanup hg19.ensGene.ra
    featureBits hg19 vegaGene
    # 64888909 bases of 2897316137 (2.240%) in intersection
    featureBits hg19 vegaPseudoGene
    # 6885145 bases of 2897316137 (0.238%) in intersection

########################################################################
# NHGRI GWAS CATALOG (DONE 11/2/11 angie)
# NEXT TIME: use snp135 and squelch multiple-mappers
# Updated 10/3/11, 9/2/11, 8/1/11, 6/9/11, 4/1/11 (last one to use snp131), 3/1/11, 2/1/11
# Updated 12/7/10, 11/1/10, 10/6/10, 9/1/10, 8/2/10, 6/2/10, 5/12/10 (last one to use snp130)
# Updated 4/1/10, 3/1/10
# Originally done 1/19/10
    mkdir /hive/data/genomes/hg19/bed/gwasCatalog
    cd /hive/data/genomes/hg19/bed/gwasCatalog
    # Done once, don't need to redo:
    zcat ../snp132/snp132.bed.gz | cut -f 1-4  \
    | sort -k4,4 \
      > snp132Coords.bed
    set today = `date +%y%m%d`
    mkdir /hive/data/genomes/hg19/bed/gwasCatalog/$today
    cd /hive/data/genomes/hg19/bed/gwasCatalog/$today

    # Mapping to hg19 by joining hg19 SNP coords with catalog flatfile (see hg18.txt)
    join -t "	" -1 4 ../snp132Coords.bed /hive/data/genomes/hg18/bed/gwasCatalog/$today/noCoords.tab \
        -o 1.1,1.2,1.3,1.4,2.2,2.3,2.4,2.5,2.6,2.7,2.8,2.9,2.10,2.11,2.12,2.13,2.14,2.15,2.16,2.17,2.18,2.19 \
    | sort -k1,1 -k2n,2n \
        > gwasCatalog.bed
    hgLoadBed hg19 gwasCatalog gwasCatalog.bed \
      -tab -sqlTable=$HOME/kent/src/hg/lib/gwasCatalog.sql -notItemRgb -allowStartEqualEnd
#Loaded 7681 elements of size 22
#Level Code Message
#Warning 1265 Data truncated for column 'initSample' at row 1995
#load of gwasCatalog did not go as planned: 7681 record(s), 0 row(s) skipped, 1 warning(s) loading bed.tab
    # See hg18.txt for notes on that warning


######################################################################## 
# ailMel1 Panda alignment (DONE - 2010-02-04 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzAilMel1.2010-02-04
    cd /hive/data/genomes/hg19/bed/lastzAilMel1.2010-02-04

    cat << '_EOF_' > DEF
# Human vs. Panda
#	parameters from the Panda paper supplemental where they describe
#	their lastz parameters
BLASTZ_K=2200
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_H=2000
BLASTZ_C=2
BLASTZ_T=2

# our usual M
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Panda
SEQ2_DIR=/scratch/data/ailMel1/ailMel1.2bit
SEQ2_LEN=/scratch/data/ailMel1/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LIMIT=50
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzAilMel1.2010-02-04
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -syntenicNet \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-chainMinScore=3000 -chainLinearGap=medium > do.log 2>&1 &
    #	real    434m21.792s
    cat fb.hg19.chainAilMel1Link.txt 
    #	1453400264 bases of 2897316137 (50.164%) in intersection

    mkdir /hive/data/genomes/ailMel1/bed/blastz.hg19.swap
    cd /hive/data/genomes/ailMel1/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzAilMel1.2010-02-04/DEF \
	-swap -noLoadChainSplit -syntenicNet \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
    real    124m14.393s
    cat fb.ailMel1.chainHg19Link.txt 
    #	1411953704 bases of 2245312831 (62.884%) in intersection

#########################################################################
# susScr1 Pig BLASTZ/CHAIN/NET (DONE - 2010-01-21 - Hiram)
    screen # use a screen to manage this multi-day job
    mkdir /hive/data/genomes/hg19/bed/lastzSusScr1.2010-01-21
    cd /hive/data/genomes/hg19/bed/lastzSusScr1.2010-01-21

    cat << '_EOF_' > DEF
# Pig vs. Human
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Pig SusScr1
SEQ2_DIR=/scratch/data/susScr1/susScr1.2bit
SEQ2_LEN=/scratch/data/susScr1/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzSusScr1.2010-01-21
TMPDIR=/scratch/tmp
'_EOF_'
    # << this line keeps emacs coloring happy

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -syntenicNet \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	-chainMinScore=3000 -chainLinearGap=medium > do.log 2>&1 &
    #	real    1072m48.949s
    cat fb.hg19.chainSusScr1Link.txt 
    #	1198793067 bases of 2897316137 (41.376%) in intersection

    mkdir /hive/data/genomes/susScr1/bed/blastz.hg19.swap
    cd /hive/data/genomes/susScr1/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzSusScr1.2010-01-21/DEF \
	-swap -noLoadChainSplit -syntenicNet \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	-chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
    #	real    119m14.040s
    cat fb.susScr1.chainHg19Link.txt 
    #	1272787231 bases of 2231332019 (57.042%) in intersection

#########################################################################
# PERSONAL GENOME VARIANTS (DONE 12/29/09 giardine)

    # This is Angie's attempt to reconstruct Belinda's steps:
    mkdir /hive/data/genomes/hg19/bed/pgSnpLiftOver
    cd /hive/data/genomes/hg19/bed/pgSnpLiftOver
    # liftOver track files in /hive/data/genomes/hg18/bed/pgSnp/
    set hg18Dir = /hive/data/genomes/hg18/bed/pgSnp
    set chainFile = /hive/data/genomes/hg18/bed/liftOver/hg18ToHg19.over.chain.gz

    #*** Are we losing insertions here?:
    foreach f (NA12878.pgSnp NA12891.pgSnp NA12892.pgSnp NA19240.pgSnp \
               pgWatson.bed pgYri3.txt pgSnpYh.txt)
      liftOver $hg18Dir/$f $chainFile $f:r.hg19.pgSnp{,.unmapped}
    end
    liftOver $hg18Dir/koref.sub.pgSnp $chainFile koref.hg19.pgSnp{,.unmapped}
    # Why pgVenter2?
    liftOver $hg18Dir/pgVenter.bed $chainFile pgVenter2.hg19.pgSnp{,.unmapped}
 
    # remove variants that are homozygous matches to hg19
cat > addRefNt.pl <<'_EOF_'
#!/usr/bin/perl -w
use strict;

my $build = 'hg19';
my $nib = "/hive/data/genomes/hg19/nib/";
my $nibFrag = "nibFrag";

while (<>) {
   chomp;
   my @f = split(/\t/);
   my $ref = '';
   if ($f[1] eq $f[2]) {
      $ref = '.'; #insertion nothing in ref
   }else {
      open(NIB, "$nibFrag $nib$f[0].nib $f[1] $f[2] + stdout |")
         or die "Couldn't run $nibFrag, $!\n";
      while(<NIB>) {
         chomp;
         if (/^>/) { next; }
         $ref .= $_;
      }
      close NIB or die "Couldn't close $nibFrag, $!\n";
   }
   #splice(@f, 3, 0, uc($ref));
   #print join("\t", @f), "\n";
   print join("\t", @f), "\t", uc($ref), "\n";
}
'_EOF_'
    # << emacs
    chmod a+x addRefNt.pl
    foreach f (*.pgSnp)
      addRefNt.pl $f \
      | perl -wpe '@w=split; s/^.*\n$// if ($w[3] eq $w[6]); s/\t\w+$//;' \
        > $f:r.filtered.pgSnp
    end
#TODO: complete attempt to reverse-engineer Belinda's work
    BelindasFix pgYri3.hg19.filtered.pgSnp > pgYri3.hg19.filtered.fixed.pgSnp

    # Load into db:
    foreach i (NA12878 NA12891 NA12892 NA19240 Watson)
      hgLoadBed -sqlTable=$HOME/kent/src/hg/lib/pgSnp.sql -renameSqlTable \
        hg19 pg$i $i.hg19.filtered.pgSnp
    end
    hgLoadBed -sqlTable=$HOME/kent/src/hg/lib/pgSnp.sql -renameSqlTable \
      hg19 pgVenter pgVenter2.hg19.filtered.pgSnp
    hgLoadBed -sqlTable=$HOME/kent/src/hg/lib/pgSnp.sql -renameSqlTable \
      hg19 pgYh1 pgSnpYh1.hg19.filtered.pgSnp
    hgLoadBed -sqlTable=$HOME/kent/src/hg/lib/pgSnp.sql -renameSqlTable \
      hg19 pgSjk koref.hg19.filtered.pgSnp
    hgLoadBed -sqlTable=$HOME/kent/src/hg/lib/pgSnp.sql -renameSqlTable \
      hg19 pgYoruban3 pgYri3.hg19.filtered.fixed.pgSnp


########################################################################
# CRG MAPABILITY (2010-01-19 - 2010-01-28, hartera, DONE)
# Data was provided by Thomas Derrien (thomas.derrien.crg.es) and Paolo Ribeca
# from the Guigo lab at the Center for Genomic Regulation (CRG) in Barcelona. 
# Data was produced using their GEM mapper aligner taking sliding k-mers 
# window of the human genome that were mapped back onto the genome with up 
# to 2mismatches. For each window, a mappability score is computed 
# S = 1/(nb of match_found) and the BigWig index was created according to 
# this score. 
# 2010-01-26 Loaded tables and added data to /gbdb/
# 2010-01-28 Changed the table names to have the "enc" prefix for consistency
# going forward with hg19 ENCODE tracks. Added trackDb entry for this 
# ENCODE Mapability track. 
# 2010-02-05 Added a 40mer sequence subtrack received on 2010-02-04.
# 2010-03-16 - 2010-03-18. Added metadata to trackDb for the subtracks and
# added downloads for the bigWig data files.
# 2010-04-28. Received new data from Thomas Derrien. Downloaded data and 
# added it to /gbdb/. A bug was found in a library used by bedGraphToBigWig so
# sent a new binary to data providers and they re-created the bigWig files. 
# 2010-05-11. All ENCODE tracks need to be preceded by the wgEncode prefix now
# on all assemblies. Update the file names in /gbdb/hg19/bbi and the 
# table names. (hartera)
# 2010-05-12. Added 24mer track to trackDb entry. Updated downloads with the
# new data.
    mkdir -p /hive/data/genomes/hg19/bed/crgMapability
    cd /hive/data/genomes/hg19/bed/crgMapability
cat << 'EOF' > temp
#!/bin/tcsh -ef
http://genome.imim.es/~tderrien/UCSC_Tracks/ALL_mappablity_hg19_H.sapiens.genome.hg19.main.mappability-36.bw.bz2
http://genome.imim.es/~tderrien/UCSC_Tracks/ALL_mappablity_hg19_H.sapiens.genome.hg19.main.mappability-50.bw.bz2
http://genome.imim.es/~tderrien/UCSC_Tracks/ALL_mappablity_hg19_H.sapiens.genome.hg19.main.mappability-75.bw.bz2
http://genome.imim.es/~tderrien/UCSC_Tracks/ALL_mappablity_hg19_H.sapiens.genome.hg19.main.mappability-100.bw.bz2
'EOF'

    awk '{if ($0 ~ /#/) print; else print "wget --timestamping \"" $0 "\"";}' \
        temp > download.csh
    rm temp
    chmod +x download.csh
    ./download.csh >& download.log &

     # Add the data to /gbdb/ and load the file names into tables (2010-01-26)
     cd /hive/data/genomes/hg19/bed/crgMapability
     bunzip2 *.bz2

     # Add data to gbdb
     mkdir -p /gbdb/hg19/bbi/
     # Symlink files with names as crgMapabilityAlignXmer.bw to /gbdb/hg18/bbi
     # and load file name into a table - one per dataset. Each table 
     # represents a subtrack.
     foreach f (`ls *.bw`)
        echo $f
        set g=`echo $f | cut -d "-" -f2`
        set num=`echo $g | cut -d "." -f1`
        set mer=`echo "${num}mer"`
        set nf=`echo "crgMapabilityAlign${mer}.bw"`
        echo $nf
        ln -s `pwd`/${f} /gbdb/hg19/bbi/${nf}
        hgsql hg19 -e "drop table if exists crgMapabilityAlign${mer}; \
     create table crgMapabilityAlign${mer} (fileName varchar(255) not null); \
     insert into crgMapabilityAlign${mer} values ('/gbdb/hg19/bbi/${nf}');"
     end

     # 2010-01-28
     # Renamed the tables to have a enc prefix for consistency going 
     # forward with hg19.
     cd /hive/data/genomes/hg19/bed/crgMapability
     hgsql -Ne 'show tables like "crg%";' hg19 > tables.txt
     foreach t (`cat tables.txt`)
        set g=`echo $t | sed -e 's/c/C/'`
        hgsql -e "alter table ${t} rename enc${g};" hg19
     end 
     # Added a trackDb entry for this ENCODE Mapability 
     # track in kent/src/hg/makeDb/trackDb/human/hg19/trackDb.enc.ra 
     # Copied track from the hg18/trackDb.wgEncode.ra entry.  
     # use bigWigInfo to check min and max values. 

     # Added data for a 40mer subtrack - 2010-02-05
     cd /hive/data/genomes/hg19/bed/crgMapability
     wget --timestamping \
http://genome.crg.es/~tderrien/UCSC_Tracks/H.sapiens.genome.hg19.main.mappability-40.bw.bz2
     bunzip2 H.sapiens.genome.hg19.main.mappability-40.bw.bz2
     ln -s `pwd`/H.sapiens.genome.hg19.main.mappability-40.bw \
/gbdb/hg19/bbi/crgMapabilityAlign40mer
        hgsql hg19 -e "drop table if exists encCrgMapabilityAlign40mer; \
     create table encCrgMapabilityAlign40mer (fileName varchar(255) not null); \
     insert into encCrgMapabilityAlign40mer values \
('/gbdb/hg19/bbi/crgMapabilityAlign40mer');"
    # Added a subtrack to trackDb/human/hg19/trackDb.enc.ra to the 
    # Mapability track.

    # 2010-03-16 - 2010-03-18
    # Added metadata to the trackDb entries for the subtracks and 
    # added downloads for these data files.
    mkdir -p /usr/local/apache/htdocs/goldenPath/hg19/encodeDCC/encMapability
    cd /usr/local/apache/htdocs/goldenPath/hg19/encodeDCC/encMapability
    cp -p /gbdb/hg19/bbi/crg*.bw .
    gzip crg*.bw
    # Copied over hg18/encodeDCC/wgEncodeMapability/preamble.html 
    # and edited it to only mention the CRG dataset. 
    # Run encodeDownloadsPage.pl to generate the index page for downloads.
    # It does not capture all the information probably because the subtrack
    # name is different to the downloads name so change the file names and 
    # re-load the tables and make the downloads. 
    cd /hive/data/genomes/hg19/bed/crgMapability
    foreach f (`ls *.bw`)
       echo $f
       set g=`echo $f | cut -d "-" -f2`
       set num=`echo $g | cut -d "." -f1`
       set mer=`echo "${num}mer"`
       set of=`echo "crgMapabilityAlign${mer}.bw"`
       set nf=`echo "encCrgMapabilityAlign${mer}.bw"`
       echo $nf
       rm /gbdb/hg19/bbi/${of}
       ln -s `pwd`/${f} /gbdb/hg19/bbi/${nf}
       hgsql hg19 -e "drop table if exists encCrgMapabilityAlign${mer}; \
     create table encCrgMapabilityAlign${mer} (fileName varchar(255) not null); \
     insert into encCrgMapabilityAlign${mer} values ('/gbdb/hg19/bbi/${nf}');"
     end
     
     cd /usr/local/apache/htdocs/goldenPath/hg19/encodeDCC/encMapability
     rm crg*
     cp -p /gbdb/hg19/bbi/encCrg*.bw .
     gzip encCrg*.bw
     # Then run encodeDownloadsPages.pl
     /cluster/home/hartera/bin/encodeDownloadsPage.pl -db=hg19 -checksum \
          -preamble=preamble.html index.html . 

     # Downloaded and added new bigWig files to /gbdb/hg19/bbi 
     # (2010-04-28 and 2010-04-30, hartera). New files were created as 
     # there was a bug in the older version of bedGraphToBigWig.          
     cd /hive/data/genomes/hg19/bed/crgMapability
     rm temp download.csh download.log 
cat << 'EOF' > temp
#!/bin/tcsh -ef
http://genome.crg.es/~tderrien/UCSC_Tracks/H.sapiens.genome.hg19.main.mappability-100.bw.bz2
http://genome.crg.es/~tderrien/UCSC_Tracks/H.sapiens.genome.hg19.main.mappability-24.bw.bz2
http://genome.crg.es/~tderrien/UCSC_Tracks/H.sapiens.genome.hg19.main.mappability-36.bw.bz2
http://genome.crg.es/~tderrien/UCSC_Tracks/H.sapiens.genome.hg19.main.mappability-40.bw.bz2
http://genome.crg.es/~tderrien/UCSC_Tracks/H.sapiens.genome.hg19.main.mappability-50.bw.bz2
http://genome.crg.es/~tderrien/UCSC_Tracks/H.sapiens.genome.hg19.main.mappability-75.bw.bz2
'EOF'
     awk '{if ($0 ~ /#/) print; else print "wget --timestamping \"" $0 "\"";}' \
         temp > download.csh
     rm temp
     chmod +x download.csh
     ./download.csh >& download.log &

     # Add data to /gbdb/. The file names in /gbdb/ are the same as before 
     # so the tables do not need to be reloaded.
     cd /hive/data/genomes/hg19/bed/crgMapability
     bunzip2 *.bz2
     foreach f (`ls *.bw`)
       echo $f
       set g=`echo $f | cut -d "-" -f2`
       set num=`echo $g | cut -d "." -f1`
       set mer=`echo "${num}mer"`
       set nf=`echo "encCrgMapabilityAlign${mer}.bw"`
       echo $nf
       rm /gbdb/hg19/bbi/${nf}
       ln -s `pwd`/${f} /gbdb/hg19/bbi/${nf}
     end

     # 2010-05-11, hartera. Re-name bigWig files and update tables
     # as all ENCODE tracks should now have the wgEncode prefix on all 
     # assemblies.
    cd /hive/data/genomes/hg19/bed/crgMapability
    foreach f (`ls *.bw`)
       echo $f
       set g=`echo $f | cut -d "-" -f2`
       set num=`echo $g | cut -d "." -f1`
       set mer=`echo "${num}mer"`
       set of=`echo "encCrgMapabilityAlign${mer}.bw"`
       set nf=`echo "wgEncodeCrgMapabilityAlign${mer}.bw"`
       echo $nf
       rm /gbdb/hg19/bbi/${of}
       ln -s `pwd`/${f} /gbdb/hg19/bbi/${nf}
       hgsql hg19 -e "drop table if exists encCrgMapabilityAlign${mer}; \
     create table wgEncodeCrgMapabilityAlign${mer} (fileName varchar(255) not null); \
     insert into wgEncodeCrgMapabilityAlign${mer} values ('/gbdb/hg19/bbi/${nf}');"
     end
     # Then change the subtrack names to match the new table names in
     # kent/src/hg/makeDb/trackDb/human/hg19/trackDb.wgEncode.ra as 
     # the contents of trackDb.enc.ra has been moved there. 

     # 2010-05-12
     # Added subtrack for the new 24mer table. 
     # Updated the downloads for the new data.
     cd /usr/local/apache/htdocs/goldenPath/hg19/encodeDCC
     # Change name of downloads directory to be consistent with the 
     # new track name.
     mv encMapability wgEncodeMapability
     cd wgEncodeMapability
     rm encCrg* md5sum.txt
     cp -p /gbdb/hg19/bbi/wgEncodeCrg*.bw .
     gzip wgEncodeCrg*.bw
     # Then run encodeDownloadsPage.pl script to update the index.html and
     # regenerate the md5sum.txt file.
     encodeDownloadsPage.pl -db=hg19 -checksum index.html 
 
#####################################################################
# tRNAs track (2010-01-13, Fan DONE)
# tRNAs track (2010-03-10, Fan RE-BUILT WITH UPDATED DATA FROM TODD LOWE)
#
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed
    mkdir tRNAs
    cd tRNAs

# Get data files from /projects/lowelab/users/lowe/Browser/vertebrates/

    cp -p /projects/lowelab/users/lowe/Browser/vertebrates/hg19-tRNAs.bed .
    cp -p \
    /projects/lowelab/users/lowe/Browser/vertebrates/hg19_tRNAs_images.tar\
    .

    hgsql hg19 -e 'drop table if exists tRNAs'
    hgLoadBed -tab hg19 tRNAs hg19-tRNAs.bed -sqlTable=$HOME/kent/src/hg/lib/tRNAs.sql

    mkdir gif
    cd gif
    tar -xvf ../hg19_tRNAs_images.tar
    mv image/* .
    rmdir image
    mkdir /hive/data/gbdb/hg19/RNA-img
    cp -p * /hive/data/gbdb/hg19/RNA-img

#####################################################################
# calJac3 Marmoset BLASTZ/CHAIN/NET (DONE - 2010-01-21 - Hiram)
    screen # use a screen to manage this multi-day job
    mkdir /hive/data/genomes/hg19/bed/lastzCalJac3.2010-02-11
    cd /hive/data/genomes/hg19/bed/lastzCalJac3.2010-02-11

    cat << '_EOF_' > DEF
# human vs. marmoset
BLASTZ=lastz
# maximum M allowed with lastz is only 254
BLASTZ_M=254
BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q
# and place those items here
BLASTZ_O=600
BLASTZ_E=150
# other parameters from panTro2 vs hg18 lastz on advice from Webb
BLASTZ_K=4500
BLASTZ_Y=15000
BLASTZ_T=2

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=20000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Marmoset (calJac3)
SEQ2_DIR=/scratch/data/calJac3/calJac3.2bit
SEQ2_LEN=/scratch/data/calJac3/chrom.sizes
SEQ2_LIMIT=50
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzCalJac3.2010-02-11
TMPDIR=/scratch/tmp
'_EOF_'
    # << this line keeps emacs coloring happy

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-syntenicNet \
	-chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    287m24.258s
    cat fb.hg19.chainCalJac3Link.txt 
    #	2047068864 bases of 2897316137 (70.654%) in intersection

    mkdir /hive/data/genomes/calJac3/bed/blastz.hg19.swap
    cd /hive/data/genomes/calJac3/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzCalJac3.2010-02-11/DEF \
	-swap -syntenicNet \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-chainMinScore=5000 -chainLinearGap=medium > swap.log 2>&1 &
    #	real    120m42.991s
    cat fb.calJac3.chainHg19Link.txt 
    #	2030475813 bases of 2752505800 (73.768%) in intersection

#############################################################################
# MAKE PCR TARGET FOR UCSC GENES (Done Feb 26, 2010 -Jim)
    ssh hgwdev
    mkdir /cluster/data/hg19/bed/mrnaPcr
    cd /cluster/data/hg19/bed/mrnaPcr
    # First, get consistent FA and PSL for UCSC Genes.
    # Initially I tried to use files from /cluster/data/hg19/bed/ucsc.10/:
    # subColumn 10 /cluster/data/hg19/bed/ucsc.10/rnaToGenome.psl
    #   /cluster/data/hg19/bed/ucsc.10/txToAcc.tab ucscGenes.hg19.psl
    # /cluster/data/hg19/bed/ucsc.10/ucscGenes.fa
    # But the psl was not from exactly the same seq's as in the fa.
    # Jim's suggestion: use sequenceForBed to get genomic-translated
    # sequences, and then genePredToFakePsl.  sequenceToBed must be
    # run on hgwdev.
    genePredToBed /cluster/data/hg19/bed/ucsc.12/ucscGenes.gp > ucscGenes.bed
    hgsql hg19 -NBe 'select kgId,geneSymbol from kgXref' \
    | perl -wpe 's/^(\S+)\t(\S+)/$1\t${1}__$2/ || die;' \
      > idSub.txt
    subColumn 4 ucscGenes.bed idSub.txt ucscGenesIdSubbed.bed
    sequenceForBed -keepName -db=hg19 -bedIn=ucscGenesIdSubbed.bed \
      -fastaOut=stdout \
    | faToTwoBit stdin kgTargetSeq.2bit
    cut -f 1-10 /cluster/data/hg19/bed/ucsc.12/ucscGenes.gp \
    | genePredToFakePsl hg19 stdin kgTargetAli.psl /dev/null

    # Load up the UCSC Genes target PSL table and put 2bit in /gbdb::
    cd /cluster/data/hg19/bed/mrnaPcr
    hgLoadPsl hg19 kgTargetAli.psl
    mkdir /gbdb/hg19/targetDb
    ln -s /cluster/data/hg19/bed/mrnaPcr/kgTargetSeq.2bit /gbdb/hg19/targetDb/

    # Ask cluster-admin to start an untranslated, -stepSize=5 gfServer on
    # /gbdb/hg19/targetDb/kgTargetSeq.2bit .

    ssh hgwdev
    # Add records to hgcentraltest blatServers and targetDb:
    hgsql hgcentraltest -e \
      'INSERT into blatServers values ("hg19Kg", "blat12", 17807, 0, 1);'
    hgsql hgcentraltest -e \
      'INSERT into targetDb values("hg19Kg", "UCSC Genes", \
         "hg19", "kgTargetAli", "", "", \
         "/gbdb/hg19/targetDb/kgTargetSeq.2bit", 1, now(), "");'

#######################################################################
# ENCODE Regulation track  (In progress March 2010, very rough instructions
# as am still defining how to build track...)

      # make root dir and link in our bad.bed file for filtering out
      # blacklisted and multiple mapping regions.
      mkdir -p /cluster/data/hg19/bed/wgEncodeReg
      cd /cluster/data/hg19/bed/wgEncodeReg
      ln -s /hive/users/kent/regulate/companion/mapping/bad.bed badRegions.bed

      # Create the DNAse peak clusters subtrack.
      # Get all of the narrowPeak format files for the wgEncodeUwDnaseSeq
      # linked into directory /hive/users/kent/regulate/dnase/peaks
      mkdir dnase
      cd dnase
      mkdir peaks
      ln -s /hive/groups/encode/dcc/analysis/ftp/pipeline/hg19/wgEncodeUwDnase/*.narrowPeak.gz peaks

      # Process these into clusters in a bed file and load clusters into
      # table.
      /bin/ls -1 peaks/*.narrowPeak.gz > peak.lst
      regClusterMakeTableOfTables uw02 peak.lst peak.table
      regCluster peak.table /dev/null peak.bed
      awk '$4 > 1 && $5 >= 100' peak.bed > wgEncodeRegDnaseClustered.bed
      hgLoadBed hg19 wgEncodeRegDnaseClustered wgEncodeRegDnaseClustered.bed

      # Make wgEncodeRegDnaseClusteredInput table.  Start with mdbQuery, and
      # then do some massaging since not completely in sync with file list.
      mdbQuery out=tab "select obj,cell,treatment,replicate,lab,dateUnrestricted from hg19 where obj like 'wgEncodeUwDnase%' and view='Peaks'" | sed 's/n\/a/None/' > inputMdb.tab
      cut -f 1 peak.table | sed 's/\.narrowPeak\.gz//' | sed 's/peaks\///' > inputs.lst
      weedLines inputs.lst inputMdb.tab wgEncodeRegDnaseClusteredInputs.tab -invert 
      hgLoadSqlTab hg19 wgEncodeRegDnaseClusteredInputs ~/kent/src/hg/lib/clusterInputDnase.sql \
	  wgEncodeRegDnaseClusteredInputs.tab

      # Make transcription factor binding site clusters.  Start with Anshul's
      # nice uniform peak calls, downloaded from 
      #   ftp://ftp-private.ebi.ac.uk/byDataType/peaks/jan2011/spp/optimal
      # using the user name and password in the ENCODE wiki at:
      #    http://genomewiki.ucsc.edu/EncodeDCC/index.php/Locations_of_ENCODE_Data
      # Put all of the ones that actually are based on two replicates into
      #    /cluster/data/hg19/bed/wgEncodeReg/tfbs/peaks
      # Once that is done do a little work towards making the cluster input.
      cd /cluster/data/hg19/bed/wgEncodeReg/tfbs
      /bin/ls peaks/*.narrowPeak.gz > peak.lst
      regClusterMakeTableOfTables ans01 peak.lst partial.table

      # Unfortunately at this point only about 90% of the peak files actually
      # have metaDb data associated with them in metaDb, so run C program that
      # takes it from metaDb if possible, otherwise tries to get it from file
      # names.
      regClusterAttachMetadataToTableOfTables hg19 partial.table withDupes.table

      # One more complication - there are often several cases where there is
      # more than one data set on the same cell+treatment+antibody combination. 
      # For good reasons hgBedToBedExps only wants one.
      # This next bit filters them out based on a picks file that was hand
      # created - picksFromDupes.  This was based on the criteria that
      # the Broad CTCF were used,  and that Snyd was used in preference
      # to OpenChromatin, which in turn was used in preference over Haib.
      # The rational for this is the Broad CTCF set has been in use for a 
      # while with no complaints,  and when examing the Pol2 signals, which
      # were with CTCF the main component of the Dupes, the Haib were the
      # weakest and the Snyd the strongest.  
      cp ~/src/hg/regulate/wgEncodeReg/picksFromDupes.tab .
      wgEncodeRegTfbsDedupeHelp withDupes.table picksFromDupes.tab peak.table

      # Make three column file: fileName, cell+treatment, antibody
      awk -F '\t' '{printf ( "%s\t%s\t%s\n", $1, ($10 == "None" ? $8 : $8 "+" $10), $9 ) ; }' peak.table > fileCellAb.tab


      # Create config file out for clustering job, using a table assigning
      # cell lines+treatments to single letter codes edited by hand.  In
      # the future may be able to automate this by having fixed upper case codes for
      # tier1&2 cell lines, and then just use first letter lower case codes
      # for rest. That is the pattern cellLetter2.tab follows anyway.
      cp ~/kent/src/hg/regulate/regClusterBedExpCfg/cellLetter2.tab .
      regClusterBedExpCfg -tabList fileCellAb.tab cluster.cfg -cellLetter=cellLetter2.tab

      # Do the actual clustering 
      hgBedsToBedExps -dupeLetterOk cluster.cfg peak.bed peak.exps

      # Load database
      hgLoadBed hg19 wgEncodeRegTfbsClustered peak.bed
      hgLoadSqlTab hg19 wgEncodeRegTfbsCells ~/kent/src/hg/lib/expRecord.sql peak.exps

      # Create inputTrackTable - six columns: <table> <source> <factor> <cell> <treatment> <lab>
      awk '{printf("%s\t%s\t%s\t%s\t%s\t%s\n", $7, ($10 == "None" ? $8 : $8 "+" $10), $9, $8, $10, $11);}' peak.table | sed 's/AlnRep/PkRep/'> wgEncodeRegTfbsClusteredInputs.tab
      hgLoadSqlTab hg19 wgEncodeRegTfbsClusteredInputs ~/kent/src/hg/lib/clusterInputTrackTable4.sql wgEncodeRegTfbsClusteredInputs.tab


      # Handle the RNA seq, starting by pooling it and filtering out multiple
      # mapping and blacklisted regions.
      mkdir -p /hive/data/genomes/hg19/bed/wgEncodeReg/txn/pooled
      cd /hive/data/genomes/hg19/bed/wgEncodeReg/txn/pooled
      bigWigMerge -adjust=-1 \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqGm12878R2x75Il200SigRep1V2.bigWig \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqGm12878R2x75Il200SigRep2V2.bigWig stdout \
	  | bedWeedOverlapping ../../badRegions.bed stdin wgEncodeRegTxnCaltechRnaSeqGm12878R2x75Il200SigPooled.bedGraph
      bigWigMerge -adjust=-1 \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqH1hescR2x75Il200SigRep1V2.bigWig \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqH1hescR2x75Il200SigRep2V2.bigWig stdout \
	  | bedWeedOverlapping ../../badRegions.bed stdin wgEncodeRegTxnCaltechRnaSeqH1hescR2x75Il200SigPooled.bedGraph
      bigWigMerge -adjust=-1 \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHct116R2x75Il200SigRep1V2.bigWig \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHct116R2x75Il200SigRep2V2.bigWig stdout \
	  | bedWeedOverlapping ../../badRegions.bed stdin wgEncodeRegTxnCaltechRnaSeqHct116R2x75Il200SigPooled.bedGraph
      bigWigMerge -adjust=-1 \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHelas3R2x75Il200SigRep1V2.bigWig \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHelas3R2x75Il200SigRep2V2.bigWig stdout \
	  | bedWeedOverlapping ../../badRegions.bed stdin wgEncodeRegTxnCaltechRnaSeqHelas3R2x75Il200SigPooled.bedGraph
      bigWigMerge -adjust=-1 \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHepg2R2x75Il200SigRep1V2.bigWig \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHepg2R2x75Il200SigRep2V2.bigWig stdout \
	  | bedWeedOverlapping ../../badRegions.bed stdin wgEncodeRegTxnCaltechRnaSeqHepg2R2x75Il200SigPooled.bedGraph
      bigWigMerge -adjust=-1 \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHsmmR2x75Il200SigRep1V2.bigWig \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHsmmR2x75Il200SigRep2V2.bigWig stdout \
	  | bedWeedOverlapping ../../badRegions.bed stdin wgEncodeRegTxnCaltechRnaSeqHsmmR2x75Il200SigPooled.bedGraph
      bigWigMerge -adjust=-1 \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHuvecR2x75Il200SigRep1V2.bigWig \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHuvecR2x75Il200SigRep2V2.bigWig stdout \
	  | bedWeedOverlapping ../../badRegions.bed stdin wgEncodeRegTxnCaltechRnaSeqHuvecR2x75Il200SigPooled.bedGraph
      bigWigMerge -adjust=-1 \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqK562R2x75Il200SigRep1V2.bigWig \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqK562R2x75Il200SigRep2V2.bigWig stdout \
	  | bedWeedOverlapping ../../badRegions.bed stdin wgEncodeRegTxnCaltechRnaSeqK562R2x75Il200SigPooled.bedGraph
      bigWigMerge -adjust=-1 \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqMcf7R2x75Il200SigRep1V2.bigWig \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqMcf7R2x75Il200SigRep2V2.bigWig stdout \
	  | bedWeedOverlapping ../../badRegions.bed stdin wgEncodeRegTxnCaltechRnaSeqMcf7R2x75Il200SigPooled.bedGraph
      bigWigMerge -adjust=-1 \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqNhekR2x75Il200SigRep1V2.bigWig \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqNhekR2x75Il200SigRep2V2.bigWig stdout \
	  | bedWeedOverlapping ../../badRegions.bed stdin wgEncodeRegTxnCaltechRnaSeqNhekR2x75Il200SigPooled.bedGraph
      bigWigMerge -adjust=-1 \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHepg2R2x75Il200SigRep1V2.bigWig \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHepg2R2x75Il200SigRep2V2.bigWig stdout \
	  | bedWeedOverlapping ../../badRegions.bed stdin wgEncodeRegTxnCaltechRnaSeqHepg2R2x75Il200SigPooled.bedGraph
      bigWigMerge -adjust=-1 \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHelas3R2x75Il200SigRep1V2.bigWig \
	  /gbdb/hg19/bbi/wgEncodeCaltechRnaSeqHelas3R2x75Il200SigRep2V2.bigWig stdout \
	  | bedWeedOverlapping ../../badRegions.bed stdin wgEncodeRegTxnCaltechRnaSeqHelas3R2x75Il200SigPooled.bedGraph
      # Now have to normalize.  This takes two passes.  The first just creates
      # a tab-separated file with normalization values. A batch file is made
      # by running awk on the tab file, and this batch file executes the
      # second half, transforming the bedGraphs to normalized values. The
      # normalization standardizes all files as if the total signal integrated to 10
      # billion over the genome.
      rm -f ../norm.tab
      foreach i (*SigPooled.bedGraph)
	  echo processing $i
	  echo -n $i " " >> ../norm.tab
	  awk '{sum += $4*($3-$2);} END {print 10000000000/sum}' $i >> ../norm.tab
      end

      awk '{printf("colTransform 4 %s 0 %g ../normalized/%s\n", $1, $2, $1);}' ../norm.tab > ../norm.csh
      chmod a+x ../norm.csh
      mkdir ../normalized
      ../norm.csh

      # Make bigWigs
      cd ../normalized
      foreach i (*.bedGraph)
	  echo processing $i
          bedGraphToBigWig $i /cluster/data/hg19/chrom.sizes $i:r.bw
      end
      
      # Link into /gbdb
      ln -s `pwd`/*.bw /gbdb/hg19/bbi

      # Tell database about it
      foreach i (*.bw)
          hgBbiDbLink hg19 $i:r /gbdb/hg19/bbi/$i
      end

      # Clean up
      cd /hive/data/genomes/hg19/bed/wgEncodeReg/txn
      rm pooled/*.bedGraph normalized/*.bedGraph

      # Make metadata: (forgive the long line)
      # JK TODO - add Hepg2 and Helas3 from here on down

      mdbQuery out=ra "select * from hg19 where \
	  obj='wgEncodeCaltechRnaSeqNhlfR2x75Il200SigRep1V2' or \
	  obj='wgEncodeCaltechRnaSeqNhekR2x75Il200SigRep1V2' or \
	  obj='wgEncodeCaltechRnaSeqH1hescR2x75Il200SigRep1V2' or \
	  obj='wgEncodeCaltechRnaSeqHelas3R2x75Il200SigRep1V2' or \
	  obj='wgEncodeCaltechRnaSeqHepg2R2x75Il200SigRep1V2' or \
	  obj='wgEncodeCaltechRnaSeqHuvecR2x75Il200SigRep1V2' or \
	  obj='wgEncodeCaltechRnaSeqHsmmR2x75Il200SigRep1V2' or \
	  obj='wgEncodeCaltechRnaSeqK562R2x75Il200SigRep1V2' or \
	  obj='wgEncodeCaltechRnaSeqGm12878R2x75Il200SigRep1V2'" \
      | sed 's/SigRep1V2/SigPooled/g' | sed 's/wgEncodeCal/wgEncodeRegTxnCal/' \
      | grep -v ^replicate > mdb.ra 

      # Now patch mdb.ra into ~/kent/src/hg/makeDb/trackDb/human/hg19/metaDb/alpha/wgEncodeReg.ra



      # At this point actually are done with this track, except for
      # all the trackDb work on the overlayed histone mod tracks.
      # The rest is for some coloring stuff that is not yet actually used.


      # Do work to figure out how to color cells.  This is based on
      # hierarchical clustering of wgEncodeUwAffyAllExon. 
      mkdir -p /hive/data/genomes/hg19/bed/wgEncodeReg/cellColors
      cd /hive/data/genomes/hg19/bed/wgEncodeReg

      # Start with some alas time consuming data cleanup to get rid of
      # overlaps - losing about 10% of the data in the process which is
      # totally ok.
      mkdir allExon
      cd allExon
      mkdir inLinks noOverlap onlyCommon
      ln -s /hive/groups/encode/dcc/analysis/ftp/pipeline/hg19/wgEncodeUwAffyExonArray/*.gz inLinks
      cd inLinks
      foreach i (*.gz)
          bedRemoveOverlap $i ../noOverlap/$i:r
      end
      cd ..
      
      # Now make up a file of regions present in all inputs for further data
      # cleanup
      bedCommonRegions noOverlap/*.* > commonExons.bed
      cd noOverlap
      foreach i (*.broadPeak)
	  echo processing $i
          bedRestrictToPositions $i ../commonExons.bed ../onlyCommon/$i
      end
      rm -r noOverlap

      # Now pool replicates where possible
      mdbQuery "select fileName,cell,replicate from hg19 where composite='wgEncodeUwAffyExonArray' and view='SimpleSignal'" -out=ra \
         | sed 's/\.gz//g' > replicate.ra
      encodeMergeReplicatesBatch replicate.ra onlyCommon merge.sh merged.ra merged
      mkdir merged
      chmod a+x merge.sh
      merge.sh

      # Resort merged output - somehow gets slightly messed by
      # encodeMergeReplicates.
      cd merged
      foreach i (*.broadPeak)
          echo sorting $i
          sort -k 1,1 -k 2,2n $i > tmp
	  mv tmp $i
      end

      # Make file that converts fileName to cell
      raToTab merged.ra -cols=fileName,cell stdout \
         | sed 's/wgEncode/merged\/wgEncode/' > fileToCell.tab

      # Now build the tree. Took 2.5 minutes
      ls -1 merged/*.broadPeak > clusterInput.lst
      regClusterTreeCells rename=fileToCell.tab clusterInput.lst cluster.tree cluster.distances




########################################################################
# DGV V10 (DATABASE OF GENOMIC VARIANTS) (DONE 11/10/10 angie - color change 2/22/11 #2917)
    ssh hgwdev
    mkdir /hive/data/genomes/hg19/bed/dgv.v10
    cd /hive/data/genomes/hg19/bed/dgv.v10
    wget --timestamping \
      http://projects.tcag.ca/variation/downloads/variation.hg19.v10.nov.2010.txt
    wget --timestamping \
      http://projects.tcag.ca/variation/downloads/indel.hg19.v10.nov.2010.txt
    # shuffle fields into bed9+ w/itemRgb
    set purple = "200,0,200"
    set red = "200,0,0"
    set blue = "0,0,200"
    set brown = "139,69,19"
    foreach f (*.v10.*.txt)
      tail -n +2 $f \
      | perl -wpe 'chomp; \
        ($id, $landmark, $chr, $start, $end, $varType, \
         undef, undef, undef, $ref, $pmid, $method, \
         $gain, $loss, undef, undef, $sample) = split("\t"); \
        $id =~ s/^Variation_//; \
        $start-- unless ($start == 0); \
        $landmark = "" if ($landmark =~ /^chr.*\d\.\.\d/); \
        $rgb = ($varType =~ /^Inv/) ? "'$purple'" : "'$brown'"; \
        if ($gain ne "" || $loss ne "") { \
          $gain =~ s/^(NA)? ?$/0/;  $loss =~ s/^(NA)? ?$/0/; \
          $rgb = "'$blue'" if ($gain > 0 && $loss == 0); \
          $rgb = "'$red'" if ($loss > 0 && $gain == 0); \
        } \
        $_ = join("\t", $chr, $start, $end, $id, 0, "+", \
                  $start, $start, $rgb, $landmark, $varType, \
                  $ref, $pmid, $method, $sample) . "\n";' \
          > $f:r.bed
    end
    hgLoadBed hg19 dgv *.bed \
      -sqlTable=$HOME/kent/src/hg/lib/dgv.sql -tab
#Loaded 101605 elements of size 15
      hgsql hg19 -NBe 'select count(distinct(pubMedId)) from dgv;'
#42

# 2/22/11 color change (Bug #2917): swap blue and red; green -> brown


#######################################################################
# DGV BETA (DATABASE OF GENOMIC VARIANTS) (DONE 10/19/11 angie)
    # DGV has changed their data format; this is a dry-run that won't be pushed
    # until the changes are no longer considered beta and the integration of
    # older studies w/new-from-dbVar/DGVa is complete.
    set today = `date +%y%m%d`
    mkdir -p /hive/data/genomes/hg19/bed/dgv/$today
    cd /hive/data/genomes/hg19/bed/dgv/$today
    wget http://dgvbeta.tcag.ca/dgv/docs/GRCh37_hg19_2011-10-17.txt
    head -1 GRCh37_hg19_2011-10-17.txt
#variantid       chr     start   end     varianttype     variantsubtype  reference       pubmedid        method  platform        mergeid frequency       samplesize      cohortdescription
    # It's more complicated than Gain/Loss/Complex or Inversion now:
    cut -f 5,6 GRCh37_hg19_2011-10-17.txt | sort | uniq -c | head -100
# 128114 CNV     ""
#  16941 CNV     CNV
#   3544 CNV     Complex
#   5574 CNV     Deletion
#   1135 CNV     Duplication
#  97916 CNV     Gain
#  42816 CNV     Insertion
# 339520 CNV     Loss
#   1805 OTHER   ""
#   5375 OTHER   Inversion
#    663 OTHER   Tandem duplication
    # shuffle fields into bed9+ w/itemRgb
    set purple = "200,0,200"
    set red = "200,0,0"
    set blue = "0,0,200"
    set brown = "139,69,19"
    tail -n +2 GRCh37_hg19_2011-10-17.txt \
    | perl -wpe 'chomp; \
      ($id, $chr, $start, $end, $varType, $varSubType, $ref, $pmid, $method, $platform, \
       undef, undef, $sampleSize, $sampleDesc) = split("\t"); \
      $start-- unless ($start == 0); \
      $landmark = ""; # was empty in DGV v10, now not even in the download file. \
      $varTypeOut = "$varType ($varSubType)"; \
      $sample = $sampleDesc; \
      $sample .= " (sample size: $sampleSize)" if ($sampleSize); \
      $method .= " ($platform)" if ($platform && $platform ne "Not Provided"); \
      $rgb = "0,0,0"; \
      if ($varType eq "CNV") { \
        if ($varSubType eq "Gain" || $varSubType eq "Insertion" || $varSubType eq "Duplication") {\
          $rgb = "'$blue'"; \
        } elsif ($varSubType eq "Loss" ||$varSubType eq "Deletion") { \
          $rgb = "'$red'"; \
        } else { \
          $rgb = "'$brown'"; \
          $varTypeOut = $varType if ($varSubType ne "Complex"); \
        } \
      } elsif ($varType eq "OTHER") { \
        if ($varSubType eq "Inversion") { \
          $rgb =  "'$purple'"; \
        } elsif ($varSubType eq "Tandem Duplication") { \
          $rgb = "'$blue'"; \
        } else { \
          $varTypeOut = $varType; \
        } \
      } \
      $_ = join("\t", "chr$chr", $start, $end, $id, 0, "+", \
                $start, $start, $rgb, $landmark, $varType, \
                $ref, $pmid, $method, $sample) . "\n";' \
        > dgv.bed
    hgLoadBed hg19 dgvBeta dgv.bed \
      -sqlTable=$HOME/kent/src/hg/lib/dgv.sql -renameSqlTable -tab


#######################################################################
# felCat4 Cat BLASTZ/CHAIN/NET (DONE  - 2010-06-07 - Chin)
    screen # use a screen to manage this multi-day job
    mkdir /hive/data/genomes/hg19/bed/lastzFelCat4.2010-06-07
    cd /hive/data/genomes/hg19/bed/lastzFelCat4.2010-06-07

    cat << '_EOF_' > DEF
# human vs. cat
# maximum M allowed with lastz is only 254
BLASTZ_M=254

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=20000000
SEQ1_LAP=10000
SEQ1_LIMIT=5

# QUERY: Cat (felCat4)
SEQ2_DIR=/scratch/data/felCat4/felCat4.2bit
SEQ2_LEN=/scratch/data/felCat4/chrom.sizes
SEQ2_LIMIT=50
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzFelCat4.2010-06-07
TMPDIR=/scratch/tmp
'_EOF_'
    # << this line keeps emacs coloring happy

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-syntenicNet -noDbNameCheck \
	-chainMinScore=3000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    # doBlastzChainNet from step chainRun after para stop, para freeBatch 
    # After para stop para freeBatch in 
    # /hive/data/genomes/hg19/bed/lastzFelCat4.2010-06-07/axtChain/run] 
    # rm the run directory,  and use memk/swarm this time

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
        `pwd`/DEF \
         -continue chainRun \
         -syntenicNet -noDbNameCheck \
         -chainMinScore=3000 -chainLinearGap=medium \
         -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
         > do_chainRun.log 2>&1 &
    # real    245m43.063s
    # *** All done !  Elapsed time: 245m43s
    # *** Make sure that goldenPath/hg19/vsFelCat4/README.txt is accurate.
    # *** Add {chain,net}FelCat4 tracks to trackDb.ra if necessary.

    cat fb.hg19.chainFelCat4Link.txt 
    #  1266003011 bases of 2897316137 (43.696%) in intersection
    # make it time independent and indicate that it is really done
    cd /hive/data/genomes/hg19/bed 
    ln -s  lastzFelCat4.2010-06-07 lastz.felCat4

    # Swap
    mkdir /hive/data/genomes/felCat4/bed/blastz.hg19.swap
    cd /hive/data/genomes/felCat4/bed/blastz.hg19.swap

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzFelCat4.2010-06-07/DEF \
	-swap -syntenicNet -noDbNameCheck \
	-workhorse=hgwdev -smallClusterHub=pk -bigClusterHub=swarm \
	-chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
    # real    432m36.917s
    cat fb.felCat4.chainHg19Link.txt 
    #   1211702270 bases of 1990635005 (60.870%) in intersection

#####################################################################
# susScr2 Pig BLASTZ/CHAIN/NET (DONE - 2010-03-26,27 - Hiram)
    screen # use a screen to manage this multi-day job
    mkdir /hive/data/genomes/hg19/bed/lastzSusScr2.2010-03-26
    cd /hive/data/genomes/hg19/bed/lastzSusScr2.2010-03-26

    cat << '_EOF_' > DEF
# Pig vs. Human
BLASTZ_M=50

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Pig SusScr2
SEQ2_DIR=/scratch/data/susScr2/susScr2.2bit
SEQ2_LEN=/scratch/data/susScr2/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzSusScr2.2010-03-26
TMPDIR=/scratch/tmp
'_EOF_'
    # << this line keeps emacs coloring happy

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -syntenicNet \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-chainMinScore=3000 -chainLinearGap=medium > do.log 2>&1 &
    #	Elapsed time: 842m23s
    cat fb.hg19.chainSusScr2Link.txt 
    #	1198794058 bases of 2897316137 (41.376%) in intersection

    mkdir /hive/data/genomes/susScr2/bed/blastz.hg19.swap
    cd /hive/data/genomes/susScr2/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzSusScr2.2010-03-26/DEF \
	-swap -noLoadChainSplit -syntenicNet \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
	-chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
    #	Elapsed time: 112m40s

    cat fb.susScr2.chainHg19Link.txt 
    #	1272785114 bases of 2231298548 (57.042%) in intersection

#########################################################################
# Vega gene update (DONE - 2010-04-07 - Hiram)
    #	lookup version number at the Vega WEB site:
    #	http://vega.sanger.ac.uk/index.html
    #	and FTP site:
    #	ftp://ftp.sanger.ac.uk/pub/vega/
    cd /hive/data/genomes/hg19
    #	step wise to verify operation
    doEnsGeneUpdate.pl -vegaGene -ensVersion=38 -stop=download hg19.ensGene.ra
    doEnsGeneUpdate.pl -vegaGene -ensVersion=38 \
	-continue=process -stop=process hg19.ensGene.ra
# genePredCheck -db=hg19 vegaPseudo.gp.gz
# checked: 11590 failed: 0
# genePredCheck -db=hg19 not.vegaPseudo.gp.gz
# checked: 96345 failed: 0
# genePredCheck -db=hg19 hg19.allGenes.gp.gz
# checked: 107935 failed: 0
    doEnsGeneUpdate.pl -vegaGene -ensVersion=38 \
	-continue=load -stop=load hg19.ensGene.ra
# zcat: download/Homo_sapiens.VEGA.38.pep.all.fa.gz: unexpected end of file
    #	they changed their file name convention ...
    doEnsGeneUpdate.pl -vegaGene -ensVersion=38 \
	-continue=cleanup hg19.ensGene.ra
    featureBits hg19 vegaGene
    # 74206453 bases of 2897316137 (2.561%) in intersection
    featureBits hg19 vegaPseudoGene
    # 8494715 bases of 2897316137 (0.293%) in intersection

#####################################################################
# oviAri1 Sheep BLASTZ/CHAIN/NET (DONE - 2010-04-16 - Chin)
    screen # use a screen to manage this multi-day job
    mkdir /hive/data/genomes/hg19/bed/lastzOviAri1.2010-04-16
    cd /hive/data/genomes/hg19/bed/lastzOviAri1.2010-04-16

    cat << '_EOF_' > DEF
# Sheep vs. Human
BLASTZ_M=50

# TARGET: Mouse Mm9
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Sheep OviAri1
SEQ2_DIR=/scratch/data/oviAri1/oviAri1.2bit
SEQ2_LEN=/scratch/data/oviAri1/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzOviAri1.2010-04-16
TMPDIR=/scratch/tmp
'_EOF_'
    # << this line keeps emacs coloring happy

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
        `pwd`/DEF \
        -noLoadChainSplit -syntenicNet \
        -workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=pk \
        -chainMinScore=3000 -chainLinearGap=medium > do.log 2>&1 &
    #   real    578m58.918s
    cat fb.hg19.chainOviAri1Link.txt
    #   878545517 bases of 2897316137 (30.323%) in intersection

    #   and the swap
    mkdir /hive/data/genomes/oviAri1/bed/blastz.hg19.swap
    cd /hive/data/genomes/oviAri1/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
        /hive/data/genomes/hg19/bed/lastzOviAri1.2010-04-16/DEF \
        -swap -noLoadChainSplit -syntenicNet \
        -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=pk \
        -chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1 &
    #   real    72m47.780s

    cat fb.oviAri1.chainHg19Link.txt
    #   824310420 bases of 1201271277 (68.620%) in intersection
    

######################################################################## 
# H-Inv 7.0 Gene track (DONE - 2010-04-07 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/hinv
    cd /hive/data/genomes/hg19/bed/hinv
    ./hinvToBed12.pl go > broken.1.exons.txt
    hgLoadBed hg19 hinv70Coding fcdna.coding.bed
    #	Loaded 272257 elements of size 12
    featureBits hg19 hinv70Coding
    #	141717797 bases of 2897316137 (4.891%) in intersection

    hgLoadBed hg19 hinv70NonCoding fcdna.nonCoding.bed
    #	Loaded 22625 elements of size 12
    featureBits hg19 hinv70NonCoding 
    #	1350960 bases of 2897316137 (0.047%) in intersection

    hgLoadBed hg19 hinv70PseudoGene fcdna.pseudoGene.bed
    #	Loaded 1166 elements of size 12
    featureBits hg19 hinv70PseudoGene
    #	1701647 bases of 2897316137 (0.059%) in intersection

    featureBits hg19 hinv70Coding hinv70PseudoGene
    #	619377 bases of 2897316137 (0.021%) in intersection

    featureBits hg19 hinv70Coding hinv70NonCoding
    #	912553 bases of 2897316137 (0.031%) in intersection

    featureBits hg19 hinv70PseudoGene hinv70NonCoding
    #	9642 bases of 2897316137 (0.000%) in intersection

######################################################################## 
# Updating the ucscToEnsembl table (DONE - 2010-04-06 - Hiram)
    #	as of Ensembl V57, their naming scheme changed for the randoms
    cd /hive/data/genomes/hg19/bed/ucscToEnsembl
cat ../../chrom.sizes | while read L
do
    size=`echo $L | awk '{print $2}'`
    ucName=`echo $L | awk '{print $1}'`
    ensName=`echo $ucName | sed -e "s/^chrM/MT/; s/^chr//;"`
    case $ucName in
        chr17_ctg5_hap1) ensName="HSCHR17_1"
                ;;
        chr4_ctg9_hap1) ensName="HSCHR4_1"
                ;;
        chr6_apd_hap1) ensName="HSCHR6_MHC_APD"
                ;;
        chr6_cox_hap2) ensName="HSCHR6_MHC_COX"
                ;;
        chr6_dbb_hap3) ensName="HSCHR6_MHC_DBB"
                ;;
        chr6_mann_hap4) ensName="HSCHR6_MHC_MANN"
                ;;
        chr6_mcf_hap5) ensName="HSCHR6_MHC_MCF"
                ;;
        chr6_qbl_hap6) ensName="HSCHR6_MHC_QBL"
                ;;
        chr6_ssto_hap7) ensName="HSCHR6_MHC_SSTO"
                ;;
        *_gl*)
ensName=`echo $L | awk '{print $1}' | sed -e "s/^chr.*_gl/GL/; s/_random//"`
                ;;
    esac
    echo -e "$ucName\t$ensName"
done > ucscToEnsemblV57.tab

    hgsql hg19 -e 'delete from ucscToEnsembl where ucsc like "%";'
    hgsql hg19 -e \
'LOAD DATA LOCAL INFILE "ucscToEnsemblV57.tab" INTO TABLE ucscToEnsembl'


############################################################################
# dbSNP BUILD 131 (SNP131) (DONE 5/25/10 angie - TWEAKED 8/4/10)
# Originally done 4/15/10 -- updated 5/25 with corrected function codes from
# dbSNP (b131_SNPContigLocusId_37_1.bcp.gz).
    # Set up build directory
    mkdir -p /hive/data/outside/dbSNP/131/{human,shared}

    # Get field encodings -- if there are changes or additions to the
    # encoding of the corresponding fields, you might need to update
    # snpNcbiToUcsc, hgTracks, hgc and hgTrackUi (see also
    # hg/lib/snp125Ui.c).
    cd /hive/data/outside/dbSNP/131/shared
    alias wg wget --timestamping
    set ftpShared = ftp://ftp.ncbi.nih.gov/snp/database/shared_data
    wg $ftpShared/LocTypeCode.bcp.gz
    wg $ftpShared/SnpClassCode.bcp.gz
    wg $ftpShared/SnpFunctionCode.bcp.gz
    wg $ftpShared/SnpValidationCode.bcp.gz
    # Here is another source -- it is not as up-to-date as the above, but
    # our encodings (enums and sets in snp131.sql) are named more similar
    # to those in the 2005 ASN:
    # ftp://ftp.ncbi.nih.gov/snp/specs/docsum_2005.asn

    ########################## DOWNLOAD #############################
    cd /hive/data/outside/dbSNP/131/human
    mkdir data schema rs_fasta
    # Get data from NCBI (anonymous FTP)
    set ftpSnpDb = ftp://ftp.ncbi.nih.gov/snp/organisms/human_9606/database
    wg ftp://ftp.ncbi.nih.gov/snp/00readme.txt
    cd /hive/data/outside/dbSNP/131/human/data
    # ContigLoc table has coords, orientation, loc_type, and refNCBI allele
    wg $ftpSnpDb/organism_data/b131_SNPContigLoc_37_1.bcp.gz
    wg $ftpSnpDb/organism_data/b131_SNPContigLocusId_37_1.bcp.gz
    wg $ftpSnpDb/organism_data/b131_ContigInfo_37_1.bcp.gz
    # MapInfo has alignment weights
    wg $ftpSnpDb/organism_data/b131_SNPMapInfo_37_1.bcp.gz
    # SNP has univar_id, validation status and heterozygosity
    wg $ftpSnpDb/organism_data/SNP.bcp.gz

    # Get schema
    cd /hive/data/outside/dbSNP/131/human/schema
    wg $ftpSnpDb/organism_schema/human_9606_table.sql.gz
    wg $ftpSnpDb/shared_schema/dbSNP_main_table.sql.gz

    # Get fasta files
    # using headers of fasta files for molType, class, observed
    cd /hive/data/outside/dbSNP/131/human/rs_fasta
    wg ftp://ftp.ncbi.nih.gov/snp/organisms/human_9606/rs_fasta/\*.gz

    ########################## LOAD NCBI TABLES #############################
    # Simplify names of data files -- strip version & extras to get
    # local canonical table names.
    cd /hive/data/outside/dbSNP/131/human/data
    foreach f (*.bcp.gz)
      set new = `echo $f \
                 | sed -e 's/^b131_SNP//; s/^b131_//; s/_37_1//; s/.bcp//;'`
      mv $f $new
      echo $new
    end

    cd /hive/data/outside/dbSNP/131/human/schema
    zcat human_9606_table.sql.gz \
    | perl -we '$/ = "\nGO\n\n\n"; \
        while (<>) { \
          next unless /^CREATE TABLE \[(b131_(SNP)?)?(ContigInfo|ContigLoc|ContigLocusId|MapInfo|SNP)(_37_1)?\]/; \
          s/b131_(SNP)?//; s/_37_1//; \
          s/[\[\]]//g;  s/GO\n\n/;/;  s/smalldatetime/datetime/g; \
          s/ON PRIMARY//g;  s/COLLATE//g;  s/Latin1_General_BIN//g; \
          s/IDENTITY (1, 1) NOT NULL /NOT NULL AUTO_INCREMENT, PRIMARY KEY (id)/g; \
          s/nvarchar/varchar/g;  s/set quoted/--set quoted/g; \
          s/(image|varchar\s+\(\d+\))/BLOB/g; \
          print; \
        }' \
      > table.sql

    # load on hgwdev (kolossus disk almost full, no more small cluster mysql5's):
    hgsql -e 'create database hg19snp131'
    cd /hive/data/outside/dbSNP/131/human/schema
    hgsql hg19snp131 < table.sql
    cd ../data

    # Avoid wasting space by excluding mappings to non-reference contigs (ContigInfo.group_label):
    zcat ContigInfo.gz | cut -f 12 | uniq | sort -u
#CRA_TCAGchr7v2
#Celera
#GRCh37
#Homo sapiens MT
#HuRef
    foreach t (ContigInfo MapInfo ContigLocusId)
      zcat $t.gz \
      | egrep -vw '(Celera|HuRef|CRA_TCAGchr7v2)' \
      | perl -wpe 's/(\d\d:\d\d:\d\d)\.0/$1/g;' \
      | hgLoadSqlTab -oldTable hg19snp131 $t placeholder stdin
    end

    # Compare contig list between our ctgPos and reference contigs in ContigInfo.
    # If they are identical, sweet, we probably have a $db/jkStuff/liftContigs.lft
    # or similar file to use below.  If they are not identical, need to make 
    # lift file using available information.
    hgsql hg19 -N -B -e 'select contig from ctgPos;' \
    | sort > /tmp/1
    # (HuRef, Celera, CRA_TCAGchr7v2 grepped out above)
    hgsql hg19snp131 -N -B -e 'select contig_acc from ContigInfo;' | sort > /tmp/2
    diff /tmp/1 /tmp/2
    # Doh!  Completely different: ctgPos has GL*, ContigInfo has NC_* / NT_*
    # We will need to generate own liftUp file for N*_* contig IDs.

    # NC_001807 entrez sez "Record removed.This sequence was removed
    # since the accepted reference sequence for the Homo sapiens
    # mitochondrion is the rCRS/Mitomap sequence, which is now
    # available as the record NC_012920".  
    # They align w/gaps on both q & t, so liftUp won't do, we need liftOver:
    blat -noHead NC_012920.fa /hive/data/genomes/hg19/M/chrM.fa stdout \
    | axtChain -psl -linearGap=medium stdin -faT NC_012920.fa /hive/data/genomes/hg19/hg19.2bit \
        NC_012920ToChrM.over.chain

    # NT_004350: entrez sez:
#COMMENT     REFSEQ INFORMATION: Features on this sequence have been produced
#            for build 37 version 1 of the NCBI's genome annotation [see
#            documentation].   The reference sequence is identical to
#            GL000003.1.

    # Using the contigs named in ContigInfo, screen-scrape genbank to get GL ID for contig ID.
    cp /dev/null contigToGl.txt
    foreach nt (`hgsql hg19snp131 -N -B -e 'select contig_acc from ContigInfo;'`)
      wget --quiet -O - 'http://www.ncbi.nlm.nih.gov/sviewer/viewer.fcgi?tool=portal&amp;db=nuccore&amp;val='$nt'&amp;dopt=genbank&amp;sendto=on' \
      | perl -we 'while (<>) { \
                    if (/^LOCUS/) { \
                      m/^LOCUS\s+'$nt'\s+(\d+) bp/ || die "parse ('$nt'): $_\t"; \
		      $size = $1; \
                    } elsif (/^            (GL\d+)\.\d+\.$/) { \
                      print "'$nt'\t$1\t$size\n"; \
		    } \
                  }' \
        >> contigToGl.txt
    end
    hgsql hg19 -NBe 'select chromStart, SUBSTRING_INDEX(contig, ".", 1), \
                       ctgPos.size, ctgPos.chrom, chromInfo.size \
                       from ctgPos,chromInfo \
                       where ctgPos.chrom=chromInfo.chrom order by contig' \
      > glToLift.txt
    sort -k2,2 contigToGl.txt \
    | join -1 2 -2 2 -t"	" -o 1.1,2.1,1.3,1.4,1.5 -a 2 -e MISSING \
        glToLift.txt - \
      > /hive/data/genomes/hg19/jkStuff/liftContigs.lft

    # Manually add NC_001807 -> chrM just in case:
    echo "0	NC_001807	16571	chrM	16571" \
      >> /hive/data/genomes/hg19/jkStuff/liftContigs.lft
    # Blat NC_012920 to chrM shows gaps, so we'll need to use liftOver chain created above.

    # Make sure there are no orient != 0 contigs among those selected.
    hgsql hg19snp131 -NBe \
      'select count(*) from ContigInfo where orient != 0;'
#0

    # ContigLoc is huge, and we want just the reference contig mappings.
    # So, based on the reference & haplo ctg_id values in ContigInfo,
    # filter to get just the mappings for those contigs:
    zcat ContigLoc.gz \
    | awk '$3 <= 9 || $3 == 6647 || $3 >= 11178' \
    | perl -wpe 's/(\d\d:\d\d:\d\d)\.0/$1/g;' \
    | hgLoadSqlTab -oldTable hg19snp131 ContigLoc placeholder stdin
#Warning 1366 Incorrect integer value: '' for column 'loc_sts_uid' at row 1
#Warning 1366 Incorrect integer value: '' for column 'loc_sts_uid' at row 2
#Warning 1366 Incorrect integer value: '' for column 'loc_sts_uid' at row 3
#Warning 1366 Incorrect integer value: '' for column 'loc_sts_uid' at row 4
#load of ContigLoc did not go as planned: 27500025 record(s), 0 row(s) skipped, 3273 warning(s) loading /dev/stdin
    zcat SNP.gz \
    | perl -wpe 's/(\d\d:\d\d:\d\d)\.0/$1/g;' \
    | hgLoadSqlTab -oldTable hg19snp131 SNP placeholder stdin
#Warning 1366 Incorrect integer value: '' for column 'CpG_code' at row 1
#Warning 1366 Incorrect integer value: '' for column 'map_property' at row 1
#Warning 1264 Out of range value adjusted for column 'last_updated_time' at row 2
#Warning 1366 Incorrect integer value: '' for column 'CpG_code' at row 2
#Warning 1366 Incorrect integer value: '' for column 'map_property' at row 2
    # ... no big deal.
    foreach t (ContigInfo ContigLoc ContigLocusId MapInfo SNP)
     echo -n "${t}:\t"
      hgsql -N -B hg19snp131 -e 'select count(*) from '$t
    end
#ContigInfo:           260
#ContigLoc:       27500025
#ContigLocusId:   55347972
#MapInfo:         23619373
#SNP:    	  23653729


    #################### EXTRACT INFO FROM NCBI TABLES ####################
    # Glom each SNP's function codes together and load up a new hg19Snp131 table.
    # Also extract NCBI's annotations of coding SNPs' effects on translation.
    # We extract ContigLocusId info only for reference assembly mapping.
    # Some SNP's functional annotations are for an alternate assembly, so we will
    # have no NCBI functional annotations to display for those (but our own are 
    # available).
    cd /hive/data/outside/dbSNP/131/human
    # Add indices to tables for a big join (5 or 6 minutes):
    hgsql hg19snp131 -e \
      'alter table ContigInfo add index (ctg_id); \
       alter table ContigLocusId add index (ctg_id);'
    hgsql hg19snp131 -NBe 'select snp_id, ci.contig_acc, asn_from, asn_to, mrna_acc, \
                           fxn_class, reading_frame, allele, residue, codon, cli.ctg_id \
                           from ContigLocusId as cli, ContigInfo as ci \
                           where cli.ctg_id = ci.ctg_id;' \
      > ncbiFuncAnnotations.txt
    wc -l ncbiFuncAnnotations.txt
#16835438 ncbiFuncAnnotations.txt
    # Ignore function code 8 (cds-reference, just means that some allele matches reference)
    # and glom functions for each SNP id:
    cut -f 1-4,6,11 ncbiFuncAnnotations.txt \
    | sort -u -k1n,1n -k6n,6n -k3n,3n -k5n,5n \
    | perl -we 'while (<>) { chomp; \
                  ($id, undef, $s, $e, $f, $c) = split; \
                  if (defined $prevId && $id == $prevId && $c == $prevC && $s == $prevS) { \
                    $prevFunc .= "$f," unless ($f == 8); \
                  } else { \
                    if (defined $prevId) { \
                      print "$prevId\t$prevC\t$prevS\t$prevE\t$prevFunc\n" if ($prevFunc); \
                    } \
                    $prevFunc = ($f == 8) ? "" : "$f,"; \
                  } \
                  ($prevId, $prevC, $prevS, $prevE) = ($id, $c, $s, $e); \
                } \
                print "$prevId\t$prevC\t$prevS\t$prevE\t$prevFunc\n" if ($prevFunc);' \
      > ucscFunc.txt
    wc -l ucscFunc.txt
#10328697 ucscFunc.txt
    cat > ucscFunc.sql <<EOF
CREATE TABLE ucscFunc (
        snp_id int NOT NULL ,
        ctg_id int(10) NOT NULL ,
        asn_from int(10) NOT NULL ,
        asn_to int(10) NOT NULL ,
        fxn_class varchar(255) NOT NULL ,
        INDEX snp_id (snp_id),
        INDEX ctg_id (ctg_id)
);
EOF
    hgLoadSqlTab hg19snp131 ucscFunc{,.sql,.txt}
    # 10/12/10: Those coords are NCBI's 0-based, fully-closed, 2-base-wide insertions.
    # We need to leave the coords alone here so ucscFunc can be joined below.
    # Make a list of SNPs with func anno's that are insertion SNPs, so we can use 
    # the list to determine what type of coord fix to apply to each annotation
    # when making snp130CodingDbSnp below.
    hgsql hg19snp131 -NBe \
      'select ci.contig_acc, cl.asn_from, cl.asn_to, uf.snp_id \
       from ucscFunc as uf, ContigLoc as cl, ContigInfo as ci \
       where uf.snp_id = cl.snp_id and \
             uf.ctg_id = cl.ctg_id and uf.asn_from = cl.asn_from and uf.asn_to = cl.asn_to and \
             cl.loc_type = 3 and \
             cl.ctg_id = ci.ctg_id' \
      > ncbiFuncInsertions.ctg.bed
    wc -l ncbiFuncInsertions.ctg.bed
#1165272 ncbiFuncInsertions.ctg.bed

    # Extract observed allele, molType and snp class from FASTA headers gnl
    # 4/13: found some inconsistent headers in rs_chPAR.fas.gz vs. other rs_ch*,
    # reported to dbSNP, Lon said that rs_chPAR.fas.gz snuck in from build 130!
    rm /hive/data/outside/dbSNP/131/human/rs_fasta/rs_chPAR.fas.gz
    zcat /hive/data/outside/dbSNP/131/human/rs_fasta/rs_ch*.fas.gz \
    | grep '^>gnl' \
    | perl -wpe 's/^\S+rs(\d+) .*mol="(\w+)"\|class=(\d+)\|alleles="([^"]+)"\|build.*/$1\t$4\t$2\t$3/ || die "Parse error line $.:\n$_\n\t";' \
    | sort -nu \
      > ucscGnl.txt
#520.305u 74.766s 7:02.48 140.8% 0+0k 0+0io 0pf+0w
    wc -l ucscGnl.txt
#23653726 ucscGnl.txt
    cut -f 1 ucscGnl.txt | uniq | wc -l
#23653726
    cat > ucscGnl.sql <<EOF
CREATE TABLE ucscGnl (
        snp_id int NOT NULL ,
        observed varchar(255) NOT NULL,
        molType varchar(255) NOT NULL,
        class varchar(255) NULL ,
        INDEX snp_id (snp_id)
);
EOF
    hgLoadSqlTab hg19snp131 ucscGnl{,.sql,.txt}

    # Add indices to tables for a big join (5 or 6 minutes):
    hgsql hg19snp131 -e \
      'alter table ContigLoc  add index (ctg_id); \
       alter table SNP        add index (snp_id); \
       alter table MapInfo    add index (snp_id);'

    # Big leftie join to bring together all of the columns that we want in snp131,
    # using all of the available joining info:
    hgsql hg19snp131 -NBe \
     'SELECT ci.contig_acc, cl.asn_from, cl.asn_to, cl.snp_id, cl.orientation, cl.allele, \
             ug.observed, ug.molType, ug.class, \
             s.validation_status, s.avg_heterozygosity, s.het_se, \
             uf.fxn_class, cl.loc_type, mi.weight, cl.phys_pos_from \
      FROM \
      ((((ContigLoc as cl JOIN ContigInfo as ci \
               ON cl.ctg_id = ci.ctg_id) \
          LEFT JOIN MapInfo as mi ON mi.snp_id = cl.snp_id and mi.assembly = ci.group_label) \
         LEFT JOIN SNP as s ON s.snp_id = cl.snp_id) \
        LEFT JOIN ucscGnl as ug ON ug.snp_id = cl.snp_id) \
       LEFT JOIN ucscFunc as uf ON uf.snp_id = cl.snp_id and uf.ctg_id = cl.ctg_id \
                                and uf.asn_from = cl.asn_from;' \
      > ucscNcbiSnp.ctg.bed
#75.815u 13.622s 32:04.35 4.6%   0+0k 0+0io 0pf+0w
    wc -l ucscNcbiSnp.ctg.bed 
#27500025 ucscNcbiSnp.ctg.bed
    # Use liftUp for everything except mito, then liftOver for mito:
    # There are some weird cases of length=1 but locType=range... in all the cases 
    # that I checked, the length really seems to be 1 so I'm not sure where they got 
    # the locType=range.  Tweak locType in those cases so we can keep those SNPs:
    grep -vw ^NC_012920 ucscNcbiSnp.ctg.bed \
    | awk -F"\t" 'BEGIN{OFS="\t";}  $2 == $3 && $14 == 1 {$14=2; numTweaked++;}  {print;} \
           END{print numTweaked, "single-base, locType=range, tweaked locType" > "/dev/stderr";}' \
    | liftUp ucscNcbiSnp.bed \
      /hive/data/genomes/hg19/jkStuff/liftContigs.lft warn stdin
#2535    single-base, locType=range, tweaked locType
#392.182u 27.358s 7:20.66 95.2%  0+0k 0+0io 0pf+0w
    # For liftOver, convert 0-base fully-closed to 0-based half-open because liftOver
    # doesn't deal with 0-base items.  Fake out phys_pos_from to 0 because many coords
    # will differ, oh well.
    grep -w NC_012920 ucscNcbiSnp.ctg.bed \
    | awk -F"\t" 'BEGIN{OFS="\t";} {$3 += 1; $16 = 0; print;}' \
    | liftOver -bedPlus=3 stdin NC_012920ToChrM.over.chain stdout chrM.unmapped \
    | awk -F"\t" 'BEGIN{OFS="\t";} {$3 -= 1; print;}' \
    | sort -k2n,2n \
      > chrMNcbiSnp.bed
#3.479u 2.428s 0:53.57 10.9%     0+0k 0+0io 4pf+0w
    # Good, got all but 2 SNPS (rs28693675 and rs55749223, partially deleted / deleted in new)
    cat chrMNcbiSnp.bed >> ucscNcbiSnp.bed
    wc -l ucscNcbiSnp.bed
#27500023 ucscNcbiSnp.bed

    # Translate NCBI's encoding into UCSC's, and perform a bunch of checks.
    # This is where developer involvement is most likely as NCBI extends the 
    # encodings used in dbSNP.
    cd /hive/data/outside/dbSNP/131/human/
    snpNcbiToUcsc ucscNcbiSnp.bed /hive/data/genomes/hg19/hg19.2bit snp131
#spaces stripped from observed:
#chr12   6093134 6093134 rs41402545
#count of snps with weight  0 = 67535
#count of snps with weight  1 = 23023681
#count of snps with weight  2 = 472416
#count of snps with weight  3 = 2536961
#count of snps with weight 10 = 1399430
#Skipped 7 snp mappings due to errors -- see snp131Errors.bed
#173.162u 5.982s 7:57.91 37.4%   0+0k 0+0io 3pf+0w
    head snp131Errors.bed
#chr13   32953907        32954033        rs80359736      rs80359736 is 126 bases long but refNCBI is different length: CATCATCAGATTTATATTCTCTGTTAACAGAAGGAAAGAGATACAGAATTTATCATCTTGCAACTTCAAAATCTAAAAGTAAATCTGAAAGAGCTAACAT
#chr17   41223118        41223133        rs80359888      Missing observed value (deleted SNP?).
#chr17   41245687        41245900        rs80359886      rs80359886 is 213 bases long but refNCBI is different length: AATATGCCTGGTAGAAGACTTCCTCCTCAGCCTATTCTTTTTAGGTGCTTTTGAATTGTGGATATTTAATTCGAGTTCCATATTGCTTATACTGCTGCTT
#chr17   41245687        41245900        rs80359886      Missing observed value (deleted SNP?).
#chr17   41276085        41276094        rs80359887      Missing observed value (deleted SNP?).
#chrM    308     310     rs66492218      Unexpected coords for locType "between" (3) -- expected NCBI's chrEnd = chrStart+1.
#chrM    308     310     rs66492218      rs66492218 is 2 bases long but refNCBI is different length: -
    wc -l snp*
#  26033053 snp131.bed
#        22 snp131.sql
#         7 snp131Errors.bed
#        18 snp131ExceptionDesc.tab
#   4281351 snp131Exceptions.bed
    # 8M new snps, lots more exceptions than snp130 (had 2631563)

    # Make one big fasta file.
    # It's a monster: 18G!  Can we split by hashing rsId?
    zcat rs_fasta/rs_ch*.fas.gz \
    | perl -wpe 's/^>gnl\|dbSNP\|(rs\d+) .*/>$1/ || ! /^>/ || die;' \
      > snp131.fa
    # Check for duplicates.
    grep ^\>rs snp131.fa | sort > /data/tmp/seqHeaders
    wc -l /data/tmp/seqHeaders
#23653726 /data/tmp/seqHeaders
    uniq /data/tmp/seqHeaders | wc -l
#23653726
    # Use hgLoadSeq to generate .tab output for sequence file offsets,
    # and keep only the columns that we need: acc and file_offset.
    # Index it and translate to snpSeq table format.
    hgLoadSeq -test placeholder snp131.fa
#23653726 sequences
#128.364u 25.531s 10:52.02 23.6% 0+0k 0+0io 0pf+0w
    cut -f 2,6 seq.tab > snp131Seq.tab
    rm seq.tab

    # Load up main track tables.
    cd /hive/data/outside/dbSNP/131/human
    hgLoadBed -tab -onServer -tmpDir=/data/tmp -allowStartEqualEnd \
      hg19 snp131 -sqlTable=snp131.sql snp131.bed
#Loaded 26033053 elements of size 17
#162.666u 19.611s 8:53.56 34.1%  0+0k 0+0io 0pf+0w
    hgLoadBed -tab -onServer -tmpDir=/data/tmp -allowStartEqualEnd \
      hg19 snp131Exceptions -sqlTable=$HOME/kent/src/hg/lib/snp125Exceptions.sql -renameSqlTable \
      snp131Exceptions.bed
#Loaded 4281351 elements of size 5
#32.020u 2.006s 1:22.87 41.0%    0+0k 0+0io 0pf+0w
    hgLoadSqlTab hg19 snp131ExceptionDesc ~/kent/src/hg/lib/snp125ExceptionDesc.sql \
      snp131ExceptionDesc.tab
    # Load up sequences.
    mkdir -p /gbdb/hg19/snp
    ln -s /hive/data/outside/dbSNP/131/human/snp131.fa /gbdb/hg19/snp/snp131.fa
    hgLoadSqlTab hg19 snp131Seq ~/kent/src/hg/lib/snpSeq.sql snp131Seq.tab

    # Put in a link where one would expect to find the track build dir...
    ln -s /hive/data/outside/dbSNP/131/human /hive/data/genomes/hg19/bed/snp131

#*** NOTE FOR NEXT TIME: ask cluster-admin to pack the snp131 table

    # Look at the breakdown of exception categories:
    cd /hive/data/outside/dbSNP/131/human
    cut -f 5 snp131Exceptions.bed | sort | uniq -c | sort -nr
#3088435 MultipleAlignments
# 886159 ObservedMismatch
#  92341 SingleClassTriAllelic
#  70184 SingleClassZeroSpan
#  43319 ObservedTooLong
#  25745 MixedObserved
#  22606 SingleClassLongerSpan
#  19681 SingleClassQuadAllelic
#  15245 FlankMismatchGenomeShorter
#   9808 DuplicateObserved
#   4463 NamedDeletionZeroSpan
#   2040 FlankMismatchGenomeLonger
#    802 ObservedContainsIupac
#    317 NamedInsertionNonzeroSpan
#    142 FlankMismatchGenomeEqual
#     62 RefAlleleMismatch
#      1 RefAlleleRevComp
#      1 ObservedWrongFormat
    # Compared to snp130, nice to see fewer disfunctional locTypes (FlankMismatch*)
    # and singleClassQuadAllelic -- major increases in most others though.
    # Sent a few bug reports to dbSNP

    # Tweaked 8/4/10 to correct missing-func (loophole fixed in perl that generates
    # ucscFunc.txt above).
    hgsql hg19 -e "update snp131 set func = 'unknown' where name = 'rs75946332' and func = '';"



############################################################################
# SPLIT SNP131 INTO CLINICAL / NON-CLINICAL (DONE 8/19/10 angie)
# http://redmine.soe.ucsc.edu/issues/559
    cd /hive/data/outside/dbSNP/131/human/data
    wget --timestamping ftp://ftp.ncbi.nlm.nih.gov/snp/database/organism_data/human_9606/SNP_bitfield.bcp.gz
    # I did a little analysis of the bitfields -- see file 
    # /hive/data/outside/dbSNP/131/human/data/bitfield_breakdown.txt .
    cd /hive/data/outside/dbSNP/131/human
    zcat data/SNP_bitfield.bcp.gz \
    | perl -wpe '@w = split;  if ($w[3] & 0x40) { $_ = "rs$w[0]\n" } else { $_ = ""; }' \
      > clinicalRsIds.txt
    wc -l clinicalRsIds.txt
#16605 clinicalRsIds.txt
    grep -Fwf clinicalRsIds.txt snp131.bed > snp131Clinical.bed
    wc -l snp131Clinical.bed
#14907 snp131Clinical.bed
    # Wow, just a subset have been mapped to hg19, bummer.
    grep -vFwf clinicalRsIds.txt snp131.bed > snp131NonClinical.bed
    wc -l snp131*Clinical.bed
#     14907 snp131Clinical.bed
#  26018146 snp131NonClinical.bed
#  26033053 total
    # Good, 26033053 is the right total.
    # Edit snp131.sql to use table name "snp131Tmp" so we don't nuke snp131.
    hgLoadBed -tab -onServer -tmpDir=/data/tmp -allowStartEqualEnd \
      hg19 snp131Clinical -sqlTable=snp131.sql -renameSqlTable snp131Clinical.bed
#Loaded 14907 elements of size 17
    hgLoadBed -tab -onServer -tmpDir=/data/tmp -allowStartEqualEnd \
      hg19 snp131NonClinical -sqlTable=snp131.sql -renameSqlTable snp131NonClinical.bed
#Loaded 26018146 elements of size 17
    # 9/1/10: Don't forget to fix the empty-func bug!
    hgsql hg19 -e 'update snp131Clinical set func = "unknown" where func = "";'


############################################################################
# SPLIT SNP131 INTO MULTIMAPPED / HAPMAP+1000GENOMES / MISC (DONE 11/5/10 angie)
# another swipe at http://redmine.soe.ucsc.edu/issues/559
    cd /hive/data/outside/dbSNP/131/human

    # First, separate out the "SNPs" that map to multiple genomic loci:
    grep -Fw MultipleAlignments snp131Exceptions.bed | wc -l
#3088435
    grep -Fw MultipleAlignments snp131Exceptions.bed \
    | cut -f 4 \
    | sort -u > multipleMappingIds.txt
    wc -l multipleMappingIds.txt
#978068 multipleMappingIds.txt
    grep -Fwf multipleMappingIds.txt snp131.bed > snp131NonUnique.bed
    wc -l snp131NonUnique.bed
#3088435 snp131NonUnique.bed
    grep -vFwf multipleMappingIds.txt snp131.bed > snp131Unique.bed
    wc -l snp131Unique.bed
#22944618 snp131Unique.bed

    # Next, separate the uniquely mapped SNPs into HapMap and/or 1000 Genomes
    # (the ones that we are certain have been observed in a large number of 
    # apparently healthy samples) vs. other ones (rarer SNPs, maybe clinical SNPs).

#*** NOTE *** To do this right, we need to get allele freq data from dbSNP and use it
#             as a filter here:

    egrep -w 'by-hapmap|by-1000genomes' snp131Unique.bed > snp131Common.bed
    wc -l snp131Common.bed
#12750453 snp131Common.bed
    egrep -vw 'by-hapmap|by-1000genomes' snp131Unique.bed > snp131Misc.bed
    wc -l snp131Misc.bed
#10194165 snp131Misc.bed

    # How many "clinical" SNPs (i.e. included in a Locus-Specific Database) are in the common set?
    grep -Fwf clinicalRsIds.txt snp131Common.bed > snp131CommonButClinical.bed
    wc -l snp131CommonButClinical.bed
#2373 snp131CommonButClinical.bed
    calc 2373 / 16605
#2373 / 16605 = 0.142909
    # A higher percentage than I expected... are 15% of OMIM / LSDB SNPs common?  
    # Spot-checking, OMIM will mention SNPs from GWAS, or as endpoints of an interval...
    # the SNPs are clearly common but sometimes common variants are associated with
    # a disorder.
    # How many "clinical" SNPs in the NonUnique set?
    grep -Fwf clinicalRsIds.txt snp131NonUnique.bed | wc -l
#901

    # Load tables:
    foreach t (snp131Common snp131Misc snp131NonUnique)
      hgLoadBed -tab -onServer -tmpDir=/data/tmp -allowStartEqualEnd \
        hg19 $t -sqlTable=snp131.sql -renameSqlTable $t.bed
    end
#Reading snp131Common.bed
#Loaded 12750453 elements of size 17
#Reading snp131Misc.bed
#Loaded 10194165 elements of size 17
#Reading snp131NonUnique.bed
#Loaded 3088435 elements of size 17


############################################################################
# ORTHOLOGOUS ALLELES IN CHIMP AND MACAQUE FOR SNP131 (DONE 6/3/10 angie)
# First done 4/15/10.  Then found that SNPs that appeared on both a main chrom
# (like chr6) and on a haplo chrom (like chr6_cox_hap2) were being flagged
# as multiple alignments when they should be, excluding them from this.
# Regenerated exceptions, then regenerated this.
    mkdir /hive/data/genomes/hg19/bed/snp131Ortho
    cd /hive/data/genomes/hg19/bed/snp131Ortho

    # Following Heather's lead in snp126orthos, filter SNPs to to keep
    # only those with class=single, length=1, chrom!~random;
    # Exclude those with exceptions MultipleAlignments,
    # SingleClassTriAllelic or SingleClassQuadAllelic.
    # Unlike snp masking, we do not filter for weight -- don't know why.
    awk '$5 ~ /^MultipleAlignments|SingleClassTriAllelic|SingleClassQuadAllelic/ {print $4;}' \
      /hive/data/outside/dbSNP/131/human/snp131Exceptions.bed \
    | sort -u \
      > snp131ExcludeIds.txt
    awk '$3-$2 == 1 && $1 !~ /_random/ && $11 == "single" {print;}' \
      /hive/data/outside/dbSNP/131/human/snp131.bed \
    | grep -vFwf snp131ExcludeIds.txt \
      > snp131Simple.bed
#333.829u 11.879s 3:57.31 145.6% 0+0k 0+0io 0pf+0w
    wc -l snp131Simple.bed
#17784981 snp131Simple.bed
#with too many SNPs excluded, was: 17337248 snp131Simple.bed

    # Glom all human info that we need for the final table onto the
    # name, to sneak it through liftOver: rsId|chr|start|end|obs|ref|strand
    awk 'BEGIN{OFS="\t";} \
        {print $1, $2, $3, \
               $4 "|" $1 "|" $2 "|" $3 "|" $9 "|" $8 "|" $6, \
               0, $6;}' \
      snp131Simple.bed > snp131ForLiftOver.bed

    # Map coords to chimp using liftOver.
    mkdir run.liftOChimp
    cd run.liftOChimp
    mkdir split out
    splitFile ../snp131ForLiftOver.bed 25000 split/chunk
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToPanTro2.over.chain.gz \
        \{check out exists out/panTro2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    ssh swarm
    cd /hive/data/genomes/hg19/bed/snp131Ortho/run.liftOChimp
    para make jobList
#Completed: 712 of 712 jobs
#CPU time in finished jobs:     127853s    2130.88m    35.51h    1.48d  0.004 y
#IO & Wait Time:                 11528s     192.14m     3.20h    0.13d  0.000 y
#Average job time:                 196s       3.26m     0.05h    0.00d
#Longest finished job:             506s       8.43m     0.14h    0.01d
#Submission to last job:           676s      11.27m     0.19h    0.01d

    # Map coords to orangutan using liftOver.
    mkdir ../run.liftOPon
    cd ../run.liftOPon
    mkdir out
    ln -s ../run.liftOChimp/split .
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToPonAbe2.over.chain.gz \
        \{check out exists out/ponAbe2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    para make jobList
# on pk:
#Completed: 712 of 712 jobs
#CPU time in finished jobs:     230882s    3848.03m    64.13h    2.67d  0.007 y
#IO & Wait Time:                  3660s      61.00m     1.02h    0.04d  0.000 y
#Average job time:                 329s       5.49m     0.09h    0.00d
#Longest finished job:            1019s      16.98m     0.28h    0.01d
#Submission to last job:          1667s      27.78m     0.46h    0.02d

    # Map coords to macaque using liftOver.
    mkdir ../run.liftOMac
    cd ../run.liftOMac
    mkdir out
    ln -s ../run.liftOChimp/split .
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToRheMac2.over.chain.gz \
        \{check out exists out/rheMac2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    para make jobList
#Completed: 712 of 712 jobs
#CPU time in finished jobs:     281168s    4686.14m    78.10h    3.25d  0.009 y
#IO & Wait Time:                 22164s     369.39m     6.16h    0.26d  0.001 y
#Average job time:                 426s       7.10m     0.12h    0.00d
#Longest finished job:             868s      14.47m     0.24h    0.01d
#Submission to last job:           872s      14.53m     0.24h    0.01d

    cd /hive/data/genomes/hg19/bed/snp131Ortho
    # Concatenate the chimp results, sorting by chimp pos in order to
    # efficiently access 2bit sequence in getOrthoSeq.  The output of
    # that is then sorted by the glommed human info field, so that we
    # can use join to combine chimp and macaque results in the next step.
    # Ditto for macaque and orangutan.  Each command pipe takes ~6 minutes:
    sort -k1,1 -k2n,2n run.liftOChimp/out/panTro2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/panTro2/panTro2.2bit \
    | sort > panTro2.orthoGlom.txt
    sort -k1,1 -k2n,2n run.liftOPon/out/ponAbe2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/ponAbe2/ponAbe2.2bit \
    | sort > ponAbe2.orthoGlom.txt
    sort -k1,1 -k2n,2n run.liftOMac/out/rheMac2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/rheMac2/rheMac2.2bit \
    | sort > rheMac2.orthoGlom.txt
    wc -l panTro2.orthoGlom.txt ponAbe2.orthoGlom.txt rheMac2.orthoGlom.txt
#  16641106 panTro2.orthoGlom.txt
#  15796202 ponAbe2.orthoGlom.txt
#  14289736 rheMac2.orthoGlom.txt
#was:  16230258 panTro2.orthoGlom.txt
#was:  15535287 ponAbe2.orthoGlom.txt
#was:  13996256 rheMac2.orthoGlom.txt

    # Use the glommed name field as a key to join up chimp and macaque
    # allele data.  Include glommed name from both files because if only
    # file 2 has a line for the key in 2.1, then 1.1 is empty.  Then plop
    # in the orthoGlom fields from each file, which are in the same order
    # as the chimp and macaque columns of snp131OrthoPanTro2RheMac2.
    join -o '1.1 2.1 1.2 1.3 1.4 1.5 1.6 2.2 2.3 2.4 2.5 2.6' \
      -a 1 -a 2 -e '?' \
      panTro2.orthoGlom.txt ponAbe2.orthoGlom.txt \
    | awk '{if ($1 != "?") { print $1, $3,$4,$5,$6,$7,$8,$9,$10,$11,$12; } \
            else           { print $2, $3,$4,$5,$6,$7,$8,$9,$10,$11,$12; }}' \
      > tmp.txt
    join -o '1.1 2.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 2.2 2.3 2.4 2.5 2.6' \
      -a 1 -a 2 -e '?' \
      tmp.txt rheMac2.orthoGlom.txt \
    | perl -wpe 'chomp; \
        ($glom12, $glom3, $o1Chr, $o1Start, $o1End, $o1Al, $o1Strand, \
         $o2Chr, $o2Start, $o2End, $o2Al, $o2Strand, \
         $o3Chr, $o3Start, $o3End, $o3Al, $o3Strand) = split; \
        $glomKey = ($glom12 ne "?") ? $glom12 : $glom3; \
        ($rsId, $hChr, $hStart, $hEnd, $hObs, $hAl, $hStrand) = \
          split(/\|/, $glomKey); \
        $o1Start =~ s/^\?$/0/;  $o2Start =~ s/^\?$/0/;  $o3Start =~ s/^\?$/0/; \
        $o1End   =~ s/^\?$/0/;  $o2End   =~ s/^\?$/0/;  $o3End   =~ s/^\?$/0/; \
        print join("\t", $hChr, $hStart, $hEnd, $rsId, $hObs, $hAl, $hStrand, \
                         $o1Chr, $o1Start, $o1End, $o1Al, $o1Strand, \
                         $o2Chr, $o2Start, $o2End, $o2Al, $o2Strand, \
                         $o3Chr, $o3Start, $o3End, $o3Al, $o3Strand) . "\n"; \
        s/^.*$//;' \
    | sort -k1,1 -k2n,2n > snp131OrthoPt2Pa2Rm2.bed
#437.114u 37.309s 6:33.92 120.4% 0+0k 0+0io 0pf+0w
    wc -l snp131OrthoPt2Pa2Rm2.bed
#17276174 snp131OrthoPt2Pa2Rm2.bed
#was: 16842459 snp131OrthoPt2Pa2Rm2.bed

    hgLoadBed -tab -onServer -tmpDir=/data/tmp -renameSqlTable \
      -sqlTable=$HOME/kent/src/hg/lib/snpOrthoPanPonRhe.sql \
      hg19 snp131OrthoPt2Pa2Rm2 snp131OrthoPt2Pa2Rm2.bed
#Loaded 17276174 elements of size 22
#123.287u 13.079s 8:17.88 27.3%  0+0k 0+0io 0pf+0w

    # Cleanup:
    nice gzip snp131Simple.bed snp131ExcludeIds.txt snp131ForLiftOver.bed
    rm -r run*/split tmp.txt *.orthoGlom.txt bed.tab


############################################################################
# DBSNP CODING ANNOTATIONS (DONE 10/12/10 angie)
# Updated 10/12/10 - redone w/corrected genome coords (Redmine Track #1249)
# Updated 5/25/10 with corrected function codes (b131_SNPContigLocusId_37_1.bcp.gz).
# Updated 4/16 - redone w/snp131, using mapping locations of dbSNP's func. annos.
#                found some strange function codes and notified dbSNP.
# originally done 6/2/09
    cd /hive/data/outside/dbSNP/131/human
    # ncbiFuncAnnotations.txt has NCBI coords: 0-based, fully closed.  
    # For anything except an insertion (0 bases between flanks), 
    # we need to add 1 to the end coord.  For an insertion, we need
    # to add 1 to the start coord.  Make a hash of the insertion IDs,
    # then look up each ID in ncbiFuncAnnotations.txt to tell which
    # transform to apply.
    # Note: sort -u with the keys below is too restrictive -- we need full line uniq.
    perl -we 'open($IDS, "ncbiFuncInsertions.ctg.bed") || die "ids: $!"; \
              while (<$IDS>) { chomp; $ids{$_} = 1; } \
              close($IDS); \
              %coding = (2=>1, 3=>1, 4=>1, 8=>1, 9=>1, 41=>1, 42=>1, 44=>1, 45=>1); \
              while (<>) { \
                chomp;  @w = split("\t"); # id, ctg, start, end, ... \
                next unless $coding{$w[5]}; \
                $bed4 = join("\t", $w[1], $w[2], $w[3], $w[0]); \
                if (exists $ids{$bed4} && $w[3] == $w[2]+1) { \
                  $w[2]++; # 2-base insertions: increment start coord \
                } else { \
                  $w[3]++; # increment end coord to get half-open \
                } \
                print join("\t", @w) . "\n"; \
              }' ncbiFuncAnnotations.txt \
    | sort -k1n,1n -k2,2 -k3n,3n -k5,5 -k6n,6n \
    | uniq \
      > ncbiCodingAnnotations.txt
    wc -l ncbiCodingAnnotations.txt
#950616 ncbiCodingAnnotations.txt

    # How many & what kinds of function types?
    cut -f 6 ncbiCodingAnnotations.txt \
    | sort -n | uniq -c
# 168639 3   (coding-synon)
# 443419 8   (cds-reference -- ignored)
#      1 9   (coding-synonymy-unknown: rs80359842)
#   9790 41  (nonsense)
# 261982 42  (missense)
#  65656 44  (frameshift)
#   1129 45  (cds-indel)
    # Gather up multiple annotation lines into one line per {snp, gene, frame}:
    perl -e  'while (<>) { chomp; \
                my ($rsId, $ctg, $s, $e, $txId, $fxn, $frm, $nt, $aa, $codon) = split("\t"); \
                if (defined $lastRs && \
                    ($lastRs != $rsId || $lastCtg ne $ctg || $lastS != $s || \
                     $lastTx ne $txId || $lastFrm ne $frm)) { \
                  if (defined $refRow) { \
                    $fxns = "$refRow->[0],$fxns";  $nts = "$refRow->[1],$nts"; \
                    $aas = "$refRow->[2],$aas";    $codons = "$refRow->[3],$codons"; \
                  } \
                  print "$lastCtg\t$lastS\t$lastE\trs$lastRs\t$lastTx\t$lastFrm\t" . \
                        "$count\t$fxns\t$nts\t$codons\t$aas\n"; \
                  $refRow = undef;  @rows = ();  ($count, $fxns, $nts, $codons, $aas) = (); \
                } \
                ($lastRs, $lastCtg, $lastS, $lastE, $lastTx, $lastFrm) = \
                    ($rsId, $ctg, $s, $e, $txId, $frm); \
                $count++; \
                if ($fxn == 8) { \
                  $refRow = [$fxn, $nt, $aa, $codon]; \
                } else { \
                 $fxns .= "$fxn,";  $nts .= "$nt,";  $aas .= "$aa,";  $codons .= "$codon,"; \
                } \
              } \
              if (defined $refRow) { \
                $fxns = "$refRow->[0],$fxns";  $nts = "$refRow->[1],$nts"; \
                $aas = "$refRow->[2],$aas";    $codons = "$refRow->[3],$codons"; \
              } \
              print "$lastCtg\t$lastS\t$lastE\trs$lastRs\t$lastTx\t$lastFrm\t" . \
                    "$count\t$fxns\t$nts\t$codons\t$aas\n";' \
      ncbiCodingAnnotations.txt \
    | liftUp snp131CodingDbSnp.bed /hive/data/genomes/hg19/jkStuff/liftContigs.lft warn stdin
    hgLoadBed hg19 snp131CodingDbSnp -sqlTable=$HOME/kent/src/hg/lib/snp125Coding.sql \
      -renameSqlTable -tab -notItemRgb -allowStartEqualEnd \
      snp131CodingDbSnp.bed
#Loaded 443454 elements of size 11


############################################################################
# SNPMASKED SEQUENCE FOR SNP131 (DONE 5/27/10 angie)
    mkdir /hive/data/genomes/hg19/snp131Mask
    cd /hive/data/genomes/hg19/snp131Mask

    # Identify rsIds with various problems -- we will exclude those.
    # MultipleAlignments is kinda broad because anything that maps on
    # both chrN and chrN_foo_hap1 will be excluded... similarly, extra
    # matches on chrN_random might disqualify good matches on chrN.
    # Well, erring on the side of caution is good.
    awk '$5 ~ /^MultipleAlignments|ObservedTooLong|ObservedWrongFormat|ObservedMismatch|MixedObserved$/ {print $4;}' \
      /hive/data/outside/dbSNP/131/human/snp131Exceptions.bed \
      | sort -u \
      > snp131ExcludeRsIds.txt
    time grep -vFwf snp131ExcludeRsIds.txt \
      /hive/data/outside/dbSNP/131/human/snp131.bed \
      > snp131Cleaned.bed
#193.507u 5.203s 4:07.62 80.2%   0+0k 0+0io 0pf+0w

    # Substitutions:
    mkdir substitutions
    snpMaskSingle snp131Cleaned.bed /hive/data/genomes/hg19/hg19.2bit stdout \
    | faSplit byname stdin substitutions/
    # 180 warnings about differing observed strings at same base position --
    # saved as diffObserved.txt.
#Masked 17377658 snps in 17377491 out of 3099287100 genomic bases
#/hive/data/genomes/hg19/hg19.2bit has 3137161264 total bases, but the total number of bases in sequences for which we masked snps is 3099287100 (difference is 37874164)
#52.903u 10.964s 4:49.08 22.0%   0+0k 0+0io 3pf+0w
    # Check that 37874164 is the total #bases in sequences with nothing in snp131Cleaned:
    grep -Fw single snp131Cleaned.bed | cut -f 1 | uniq > /tmp/1
    grep -vwf /tmp/1 ../chrom.sizes
    grep -vwf /tmp/1 ../chrom.sizes \
    | awk 'BEGIN {TOTAL = 0;}  {TOTAL += $2;}  END {printf "%d\n", TOTAL;}'
#37874164
#TODO: send list to dbSNP.
    # Make sure that sizes are identical, first diffs are normal -> IUPAC,
    # and first diffs' case is preserved:
    foreach f (substitutions/chr*.fa)
      faCmp -softMask $f ../[1-9UMXY]*/$f:t |& grep -v "that differ"
    end
#chr1 in substitutions/chr1.fa differs from chr1 at ../1/chr1.fa at base 10491 (y != c)
#chr10 in substitutions/chr10.fa differs from chr10 at ../10/chr10.fa at base 61004 (r != a)
#...
#(output OK -- ambiguous bases replacing [agct] at SNP positions)
    foreach f (substitutions/chr*.fa)
      echo $f:t:r
      mv $f $f:r.subst.fa
      gzip $f:r.subst.fa
    end

    # Insertions:
    mkdir insertions
    snpMaskAddInsertions snp131Cleaned.bed /hive/data/genomes/hg19/hg19.2bit stdout \
    | faSplit byname stdin insertions/
#Added 2496221 snps totaling 5939697 bases to 3098816404 genomic bases
#/hive/data/genomes/hg19/hg19.2bit has 3137161264 total bases, but the total number of bases in sequences for which we masked snps is 3098816404 (difference is 38344860)
#52.764u 12.593s 3:55.47 27.7%   0+0k 0+0io 2pf+0w
    # Again, that just means that some chroms didn't have filtered SNPs.
    # Make sure that all sizes have increased relative to original:
    foreach f (insertions/chr*.fa)
      echo -n "${f:t:r}: "
      faCmp -softMask $f ../[1-9UMXY]*/$f:t |& grep -v "that differ" \
      |& perl -we '$_=<>; \
           if (/^\w+ in \S+ has (\d+) bases.  \w+ in \S+ has (\d+) bases/) { \
             if ($1 > $2) {print "OK: ins size $1 > $2\n";} \
             else {die "ERROR: ins size $1 <= $2\n";} \
           } else {die $_;}'
    end
#chr1: OK: ins size 249717078 > 249250621
#chr10: OK: ins size 135805198 > 135534747
#...
#(output OK -- new sizes > old)
    foreach f (insertions/chr*.fa)
      mv $f $f:r.ins.fa
      gzip $f:r.ins.fa
    end

    # Deletions:
    mkdir deletions
    snpMaskCutDeletions snp131Cleaned.bed /hive/data/genomes/hg19/hg19.2bit stdout \
    | faSplit byname stdin deletions/
#Cut 1522178 snps totaling 3455905 bases from 3098701788 genomic bases
#/hive/data/genomes/hg19/hg19.2bit has 3137161264 total bases, but the total number of bases in sequences for which we masked snps is 3098701788 (difference is 38459476)
#114.251u 20.911s 4:24.26 51.1%  0+0k 0+0io 3pf+0w
    # Again, that just means that some chroms didn't have filtered SNPs.
    # Make sure that all sizes have decreased relative to original:
    foreach f (deletions/chr*.fa)
      echo -n "${f:t:r}: "
      faCmp -softMask $f ../[1-9UMXY]*/$f:t |& grep -v "that differ" \
      |& perl -we '$_=<>; \
           if (/^\w+ in \S+ has (\d+) bases.  \w+ in \S+ has (\d+) bases/) { \
             if ($1 < $2) {print "OK: del size $1 < $2\n";} \
             else {die "ERROR: del size $1 >= $2\n";} \
           } else {die $_;}'
    end
#chr1: OK: del size 248968549 < 249250621
#chr10: OK: del size 135378065 < 135534747
#...
#(output OK -- del sizes < old)
    foreach f (deletions/chr*.fa)
      mv $f $f:r.del.fa
      gzip $f:r.del.fa
    end

    # Clean up and prepare for download:
    gzip snp131Cleaned.bed
    foreach d (substitutions insertions deletions)
      pushd $d
        md5sum *.gz > md5sum.txt
        cp /hive/data/genomes/hg18/snp130Mask/$d/README.txt .
      popd
    end
    # Edit the README.txt in each subdir.

    # Create download links on hgwdev.
    # NOTE: Currently we offer only the substitutions.
    # If we get any user requests, then maybe we can put the insertions
    # and deletions out there.
    mkdir /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/snp131Mask
    ln -s /hive/data/genomes/hg19/snp131Mask/substitutions/* \
      /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/snp131Mask/
## If there is user demand for ins & del, then start over with an empty
## goldenPath/snp131Mask and do this:
##    foreach type (substitutions insertions deletions)
##      mkdir /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/snp131Mask/$type
##      ln -s /hive/data/genomes/hg19/snp131Mask/$type/* \
##        /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/snp131Mask/$type/
##    end


##############################################################################
#  RE-BUILD sno/miRNA TRACK (DONE - 04-20-2010 - Chin)

    # The data in this track is out of date so update the track. 
    mkdir -p /hive/data/genomes/hg19/bed/wgRna-2010-04-20
    cd /hive/data/genomes/hg19/bed/wgRna-2010-04-20

    # Download GFF file of latest miRNA annotations from miRBase at the
    # ftp://mirbase.org/pub/mirbase/CURRENT/. This is Release 14.0
    # (September,  2009)
    # 04-27-2010 Get the newest miRNA release 15
    wget --timestamping \
         ftp://mirbase.org/pub/mirbase/CURRENT/genomes/hsa.gff
    # Re-format, need to add "chr" to the beginning of each line.
    sed -e 's/^/chr/' hsa.gff > hsMirBaseFormat.gff
    # Remove extra "chr" in comment lines
    perl -pi.bak -e 's/chr#/#/' hsMirBaseFormat.gff
    # Change chrMT to chrM
    perl -pi.bak -e 's/chrMT/chrM/' hsMirBaseFormat.gff
    # Remove all but ID name in last field
    sed -e 's/\";//g' hsMirBaseFormat.gff | sed -e 's/ID=\"//g' \
       | sed -e 's/ACC=\"MI[0-9]*\s//' > hsMirBaseFormatIdOnly.gff

    # set score to zero, since the color is based on the type of the RNA
    # Starts appear to be 1-based when compared to miRNAs in current
    # track
    # and those in Ensembl.
    # Confirmed with Sam Griffith-Jones (one of the authors of miRBase,
    # sam.griffith-jones@manchester.ac.uk) that these GFF coordinates
    # are 1-based. 
    # Also add thickStart and thickEnd columns and "miRNA" for type.
    awk 'BEGIN {FS="\t"} {OFS="\t"} \
        {if ($0 !~ /#/ && $7 == "+") \
         print $1, $4-1, $5, $9, 0, $7, 0, 0, "miRNA"; \
       else if ($0 !~ /#/ && $7 == "-") \
         print $1, $4-1, $5, $9, 0, $7, 0, 0, "miRNA";}' \
        hsMirBaseFormatIdOnly.gff > hsMirBaseFormatIdOnly.bed

    # 2010-04-21
    # Down load the current snoRNABase coordinates (version 3, based on hg19)
    #  from 
    # http://www-snorna.biotoul.fr/coordinates.php
    #   to
    # /hive/data/genomes/hg19/bed/wgRna-2010-04-20/snoRNABaseVer3Coords.xls

    cd /hive/data/genomes/hg19/bed/wgRna-2010-04-20/
    cp snoRNABaseVer3Coords.xls snoRNABaseVer3Coords.txt
    # remove the header line (column title).
    # remove all the quotes surrounding characters field
    perl -pi.bak -e 's/\"//g' snoRNABaseVer3Coords.txt
    # Reformat to BED format with thickStart and thickEnd set to 0.
    awk 'BEGIN {FS="\t"} {OFS="\t"} \
        {if ($4 == "+") \
         print $1, $2-1, $3, $5, 0, $4, 0, 0,$6; \
       else if ($4 == "-") \
         print $1, $2-1, $3, $5, 0, $4, 0, 0,$6;}' \
       snoRNABaseVer3Coords.txt > snoRNABaseVer3Coords.bed
    # 2010-08-02: snoRnaBase team has not response to hg19 update request.
    # use liftOver to convert the 400 coordinates to hg19 directly.
    liftOver snoRNABaseVer3Coords.bed -bedPlus=3 \ 
      /hive/data/genomes/hg18/bed/liftOver10K/hg18ToHg19.over.chain.gz \
        snoRNABaseHg19Coords.bed unMapped
    # Reading liftover chains
    # Mapping coordinates



    # Merge the miRNA and snoRNA files together
    cat hsMirBaseFormatIdOnly.bed snoRNABaseHg19Coords.bed \
        > wgRna20100420.bed
    # Create and load wgRna
    cp -p /cluster/bin/build/build-kent/src/hg/lib/wgRna.sql wgRna.sql
    hgLoadBed -sqlTable=wgRna.sql hg19 wgRna wgRna20100420.bed
    #  Reading wgRna20100420.bed
    #  Loaded 1341 elements of size 9
    #  Sorted
    #  Creating table definition for wgRna
    #  Saving bed.tab
    #  Loading hg19

    # Clean up
    rm *.bak

    # some details about this track:
    hgsql -e "select count(*) from wgRna;" hg19
    #  1341
    # contain 4 types:
    cat wgRna20100420.bed | awk '{print $9}' | sort | uniq
    # CDBox
    # HAcaBox
    # miRNA
    # scaRna
    hgsql -e "select type, count(*) from wgRna  group by type;" hg19
    #   CDBox     269
    #   HAcaBox   112
    #   miRNA     939
    #   scaRna     21
    featureBits hg19 wgRna
    # 122226 bases of 2899183193 (0.004%) in intersection


#############################################################################
# AFFY U133Plus2 (DONE 2010-10-04 Chin) (Removed prefixes 2010-11-29 galt)
    # Align probes 
    ssh swarm
    cd /hive/data/genomes/hg19/bed
    mkdir -p affyProbes/affyU133Plus2/run
    cd affyProbes/affyU133Plus2/run
    mkdir psl
    ls -1 /scratch/data/hg19/nib/*.nib > genome.lst
    ls -1 /hive/data/outside/affyProbes/U133Plus2_all.fa > mrna.lst

    cat << '_EOF_' > gsub
#LOOP
/cluster/bin/x86_64/blat -fine -ooc=/scratch/data/hg19/11.ooc  $(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    gensub2 genome.lst mrna.lst gsub jobList
    para create jobList
    para try
    para check
    para push
    para time
    # Completed: 96 of 96 jobs
    # CPU time in finished jobs: 31136s     518.93m     8.65h    0.36d  0.001 y
    # IO & Wait Time:            2218s      36.97m     0.62h    0.03d  0.000 y
    # Average job time:                 347s       5.79m     0.10h    0.00d
    # Longest finished job:            2548s      42.47m     0.71h    0.03d
    # Submission to last job:          4244s      70.73m     1.18h    0.05d

    # Do sort, best in genome filter.
    # to create affyU133Plus2.psl.
    pslSort dirs raw.psl tmp psl
    pslReps -minCover=0.3 -minAli=0.95 -nearTop=0.005 raw.psl ../affyU133Plus2.psl /dev/null
    #   Processing raw.psl to ../affyU133Plus2.psl and /dev/null
    #   .....Processed 693340 alignments
    rm -r raw.psl psl

    # Load probes and alignments into database.
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/affyProbes/affyU133Plus2

    # remove prefix
    perl -pi.bak -e "s/U133\+2://" affyU133Plus2.psl
    hgLoadPsl hg19 affyU133Plus2.psl
    hgLoadSeq -abbr=U133+2: hg19 /gbdb/hgFixed/affyProbes/U133Plus2_all.fa
         # note: re-ran the -replace was used with hgLoadSeq
    #   Creating seq.tab file
    #   Adding /gbdb/hgFixed/affyProbes/U133Plus2_all.fa
    #   54613 sequences
    #   Updating seq table
    #   All done
    #  hgsql -e "select count(*) from affyU133Plus2;" hg19
    #     58592  


    # Added knownToU133Plus2 table
    hgMapToGene hg19 affyU133Plus2 knownGene knownToU133Plus2


#############################################################################
# AFFY U95 (DONE 2010-10-07 Chin)  (Removed prefixes 2010-11-29 galt)
    # Align probes 
    ssh swarm
    cd /hive/data/genomes/hg19/bed
    mkdir -p affyProbes/affyU95/run
    cd affyProbes/affyU95/run
    mkdir psl
    ls -1 /scratch/data/hg19/nib/*.nib > genome.lst
    ls -1 /hive/data/outside/affyProbes/HG-U95Av2_all.fa > mrna.lst

    cat << '_EOF_' > gsub
#LOOP
/cluster/bin/x86_64/blat -fine -ooc=/scratch/data/hg19/11.ooc  $(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    gensub2 genome.lst mrna.lst gsub jobList
    para create jobList
    para try
    para check
    para push
    para time
# Completed: 93 of 93 jobs
# CPU time in finished jobs:       2101s      35.01m     0.58h    0.02d  0.000 y
# IO & Wait Time:                   657s      10.95m     0.18h    0.01d  0.000 y
# Average job time:                  30s       0.49m     0.01h    0.00d
# Longest finished job:             165s       2.75m     0.05h    0.00d
# Submission to last job:           619s      10.32m     0.17h    0.01d
# Estimated complete:                 0s       0.00m     0.00h    0.00d
#Submission to last job:          1685s      28.08m     0.47h    0.02d


    # Do sort, best in genome filter.
    # to create affyU95.psl.
    pslSort dirs raw.psl tmp psl
    pslReps -minCover=0.3 -minAli=0.95 -nearTop=0.005 raw.psl ../affyU95.psl /dev/null
    rm -r raw.psl psl

    # Load probes and alignments into database.
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/affyProbes/affyU95

    # remove prefix
    perl -pi.bak -e "s/U95Av2://" affyU95.psl
    hgLoadPsl hg19 affyU95.psl
    hgLoadSeq -abbr=U95Av2: hg19 /gbdb/hgFixed/affyProbes/HG-U95Av2_all.fa
         # note: re-ran the -replace was used with hgLoadSeq
    #   Creating seq.tab file
    #   Adding /gbdb/hgFixed/affyProbes/HG-U95Av2_all.fa
    #   12386 sequences
    #   Updating seq table
    #   All done

    # Added knownToU95 table
    hgMapToGene hg19 affyU95 knownGene knownToU95

#############################################################################
# ucscRetro track (2010-03-31, baertsch,hartera DONE)
mkdir -p /hive/groups/gencode/pseudogenes/retroFinder/retro/hg19
cd /hive/groups/gencode/pseudogenes/retroFinder/retro/hg19

mkdir -p /hive/data/genomes/hg19/bed/retro/ 

cat << '_EOF_' > DEF

RETRO_OPTIONS="-verbose=4 -minAli=0.98 -nearTop=0.005 -skipBlatMerge "
DB=hg19
SCORETHRESH=550
GENOMENAME='Homo sapiens'
GBDB=hg
MRNABASE=/hive/data/genomes/$DB/bed/mrnaBlastz
TMPMRNA=/hive/groups/gencode/pseudogenes/retroFinder/mrnaBlastz/$DB
TMPEST=/hive/groups/gencode/pseudogenes/retroFinder/est/$DB
EST=all_est
SPLICED_EST=intronEst
SPLIT_EST=0
SPLIT_SPLICED_EST=0
SCRIPT=/cluster/home/baertsch/baertsch/scripts
GENOME=/hive/data/genomes/
RETRODIR=$GENOME/$DB/bed/retro
BASE=/hive/groups/gencode/pseudogenes/retroFinder/retro
OUTDIR=/hive/groups/gencode/pseudogenes/retroFinder/retro/$DB
RESULT=$OUTDIR/result
LOG=$OUTDIR/log
OUT=$OUTDIR/out
OVERLAPDIR=$OUTDIR/run.o
VERSION=2
TABLE=ucscRetroInfo$VERSION
ALIGN=ucscRetroAli$VERSION
LOCAL=/scratch/data/$DB
NIB=$LOCAL/nib
RMSK=rmsk
NET1=netCanFam2 
NET2=netMm9 
NET3=netRheMac2 
GENE1=knownGene
GENE2=refGene
GENE3=mgcGenes
CLUSTER=swarm
SPECIES="hg19"
ROOTDIR="/cluster/home/hartera/public_html"
EXPDIR=exp
GENEPFAM=knownGene
PFAM=knownToPfam
PFAMIDFIELD=name
PFAMDOMAIN=value
ARRAY=gnfAtlas2
ARRAYMEDIAN=hgFixed.gnfHumanAtlas2Median
ARRAYRATIO=hgFixed.gnfHumanAtlas2AllRatio
ARRAYABS=hgFixed.gnfHumanAtlas2All
ARRAYEXP=hgFixed.gnfHumanAtlas2MedianExps
ARRAYEXPALL=hgFixed.gnfHumanAtlas2AllExps
ARRAYLOOKUP=knownToGnfAtlas2
ARRAYPSLS="/hive/data/genomes/hg19/bed/geneAtlas2/affyU133A.psl /hive/data/genomes/hg19/bed/geneAtlas2/affyGnf1h.psl"
ALTSPLICE=
SPLITBYAGE=$SCRIPT/splitRetrosByAge
PDB=proteins090821

'_EOF_'
    # << happy emacs

retroFinder-1.16/scripts/filterMrna.sh DEF
retroFinder-1.16/scripts/filterEst.sh DEF
nohup retroFinder-1.16/scripts/ucscRetroStep1.sh DEF
#check cluster job
nohup retroFinder-1.16/scripts/ucscRetroStep2.sh DEF
nohup retroFinder-1.16/scripts/ucscRetroStep3.sh DEF
#check cluster job
nohup retroFinder-1.16/scripts/ucscRetroStep4.sh DEF
    # Load the track
nohup retroFinder-1.16/scripts/ucscRetroStep5.sh DEF
nohup retroFinder-1.16/scripts/ucscRetroStep6.sh DEF
#add ucscRetroAli to trackDb.ra 
cp  ucscRetroAli.psl /hive/data/genomes/hg19/bed/retro/ 
cp  ucscRetroInfo.bed /hive/data/genomes/hg19/bed/retro/ 
cp  ucscRetroCds.tab /hive/data/genomes/hg19/bed/retro/ 

cd /hive/data/genomes/hg19/bed/retro/ 
#############################################################################
# UPDATE KEGG TABLES (DONE, Fan, 6/18/10)

mkdir -p /hive/data/genomes/hg19/bed/pathways/kegg
cd /hive/data/genomes/hg19/bed/pathways/kegg

wget --timestamping ftp://ftp.genome.jp/pub/kegg/pathway/map_title.tab

cat map_title.tab | sed -e 's/\t/\thsa\t/' > j.tmp
cut -f 2 j.tmp >j.hsa
cut -f 1,3 j.tmp >j.1
paste j.hsa j.1 |sed -e 's/\t//' > keggMapDesc.tab
rm j.hsa j.1
rm j.tmp

hgsql hg19 -e 'drop table keggMapDesc'
hgsql hg19 < ~/kent/src/hg/lib/keggMapDesc.sql
hgsql hg19 -e 'load data local infile "keggMapDesc.tab" into table keggMapDesc'

wget --timestamping ftp://ftp.genome.jp/pub/kegg/genes/organisms/hsa/hsa_pathway.list

cat hsa_pathway.list| sed -e 's/path://'|sed -e 's/:/\t/' > j.tmp
hgsql hg19 -e 'drop table keggPathway'
hgsql hg19 < ~/kent/src/hg/lib/keggPathway.sql
hgsql hg19 -e 'load data local infile "j.tmp" into table keggPathway'

hgsql hg19 -N -e \
'select name, locusID, mapID from keggPathway p, knownToLocusLink l where p.locusID=l.value' \
>keggPathway.tab

hgsql hg19 -e 'delete from keggPathway'

hgsql hg19 -e 'load data local infile "keggPathway.tab" into table keggPathway'

rm j.tmp
#############################################################################
# Add KEGG column to hg19 Gene Sorter (Done, Fan, 6/18/2010)

mkdir -p /hive/data/genomes/hg19/bed/geneSorter
cd /hive/data/genomes/hg19/bed/geneSorter
hgsql hg19 -N -e 'select kgId, mapID, mapID, "+", locusID from keggPathway' |sort -u|sed -e 's/\t+\t/+/' > knownToKeggEntrez.tab

hgsql hg19 -e 'drop table knownToKeggEntrez'

hgsql hg19 < ~/kent/src/hg/lib/knownToKeggEntrez.sql

hgsql hg19 -e 'load data local infile "knownToKeggEntrez.tab" into table knownToKeggEntrez'

#############################################################################
# Haplotype locations (DONE - 2010-06-29 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/haplotypeLocations
    cd /hive/data/genomes/hg19/bed/haplotypeLocations
    for H in 1 2 3 4 5 6 7 8 9
do
    grep -v "^#" ../../download/alternate_loci/ALT_REF_LOCI_${H}/placed_scaffolds/alt_locus_scaf2primary.pos | awk -F'\t' '
{
printf "chr%d\t%d\t%d\t%s\n", $3, $4, $6, $1
}
'
done | sed -e "s/HSCHR6_MHC_APD_CTG1/chr6_apd_hap1/;
s/HSCHR6_MHC_COX_CTG1/chr6_cox_hap2/;
s/HSCHR6_MHC_DBB_CTG1/chr6_dbb_hap3/;
s/HSCHR6_MHC_MANN_CTG1/chr6_mann_hap4/;
s/HSCHR6_MHC_MCF_CTG1/chr6_mcf_hap5/;
s/HSCHR6_MHC_QBL_CTG1/chr6_qbl_hap6/;
s/HSCHR6_MHC_SSTO_CTG1/chr6_ssto_hap7/;
s/HSCHR4_1_CTG9/chr4_ctg9_hap1/;
s/HSCHR17_1_CTG5/chr17_ctg5_hap1/;" > haplotypeLocations.bed

    hgLoadBed hg19 haplotypeLocations haplotypeLocations.bed
    featureBits hg19 haplotypeLocations
# 7207422 bases of 2897316137 (0.249%) in intersection

#############################################################################
# BUILD THE TRACK OF IKMC MAPPED TO HUMAN GENOME. (DONE, Fan, 8/2/11)

    ssh hgwdev
    mkdir -p /hive/data/genomes/hg19/bed/ikmc/110801
    cd /hive/data/genomes/hg19/bed/ikmc/110801

# recieve  20110801_human.gff.gz from Carol Bult [Carol.Bult@jax.org] and place it under this subdirectory.

# build hgIkmc table from raw data file ucschuman.gff (substitue some
# troublesome chroms in raw data file and remove some records mapped to 'chrUn')

    zcat 20110801_human.gff.gz |sort -u \
    |sed -e 's/chr9|NT_113911.1/chr9/' \
    |grep -v 'chrUn' \
    | perl -we \
      'while (<>) { \
         s/\r?\n$//; \
         ($chr, undef, $ctr, $s, $e, undef, undef, undef, $id, $col, $n) = split("\t"); \
         if ($s eq "") { warn "$_\n";  s/^.*//; next; } # Some lines have no coords. \
         $col = ($col eq "Yellow") ? "255,215,0" : \
                ($col eq "Green")  ? "0,240,0" : \
                ($col eq "Blue")   ? "0,0,200" : "0,0,0"; \
         $s--; \
         $id =~ s/^MGI:\d+; (\w+); .*/$1/ || die "Cant parse id \"$id\""; \
         my $geneId = join("|", $chr, $ctr, "${n}_$id"); \
         push @{$geneBlks{$geneId}}, [$s, $e, $col] unless $e <= 0; \
      } \
      warn "Got " . scalar(keys %geneBlks) . " genes.\n"; \
      foreach my $geneId (keys %geneBlks) { \
         my @blks = @{$geneBlks{$geneId}}; \
         my ($chrom, $center, $name) = split(/\|/, $geneId); \
         my $blkCount = @blks; \
         @blks = sort {$a->[0] <=> $b->[0]} @blks; \
         my $chromStart = $blks[0]->[0]; \
         my $chromEnd = $blks[$blkCount-1]->[1]; \
         my $color = $blks[0]->[2]; \
         my $blkStarts = ""; \
         my $blkSizes = ""; \
         foreach my $blk (@blks) { \
           my ($start, $end, $col) = @{$blk}; \
           $blkStarts .= ($start - $chromStart) . ","; \
           $blkSizes  .= ($end - $start) . ","; \
           if ($col ne $color) { die "Blocks of $geneId of colors $color and $col"; } \
         } \
        print join("\t", $chrom, $chromStart, $chromEnd, $name, 0, ".", $chromStart, \
                   $chromStart, $color, $blkCount, $blkSizes, $blkStarts) . "\n"; \
      }' \
    | sort -k 1,1 -k 2n,2n > hgIkmc.bed
    
# build hgIkmcExtra table

    zcat 20110801_human.gff.gz \
    | grep -v 'chrUn' \
    | perl -wpe 's/\r?\n$//; @w = split("\t"); \
      if ($w[3] eq "") { s/^.*//; next; } # Some lines have no coords. \
      if ($w[4] <= 0) { s/^.*//; next; } # A few lines have end=0. \
      $w[8] =~ m/^(MGI:\d+); (\w+); (\w.*)/ || die; \
      ($mgi, $designId, $status) = ($1, $2, $3); \
      $_ = "$w[10]_$designId\t$mgi,$designId,$w[2],$status\n";' \
    | sort -u > hgIkmcExtra.tab
    wc -l hgIkmcExtra.tab

# load tables
    hgLoadBed hg19 hgIkmc hgIkmc.bed
    checkTableCoords  -verbose=2  hg19 hgIkmc

    hgLoadSqlTab hg19 hgIkmcExtra $HOME/kent/src/hg/lib/genericAlias.sql hgIkmcExtra.tab

#############################################################################
# adding patches to the sequence (DONE - 2010-07-23)
    #	fetch the "official" chrM sequence
    mkdir -p /hive/data/genomes/hg19/bed/additionalSequence/chrM
    cd /hive/data/genomes/hg19/bed/additionalSequence/chrM
    wget --timestamping -O NC_012920.1.fa \
"http://www.ncbi.nlm.nih.gov/sviewer/viewer.fcgi?db=nuccore&dopt=fasta&sendto=on&id=NC_012920.1.fa"
    echo ">chrM_NC_012920" > chrM_NC_012920.fa
    grep -v "^>" NC_012920.1.fa | sed -e "/^$/d" >> chrM_NC_012920.fa

    #	fetch the first two patches:
    mkdir -p /hive/data/genomes/hg19/bed/additionalSequence/patches
    cd /hive/data/genomes/hg19/bed/additionalSequence/patches
    wget --cut-dirs=7 --no-parent --timestamping --no-remove-listing -m \
        -nH --ftp-user=anonymous --ftp-password=yourEmail@your.domain \
"ftp://ftp.ncbi.nlm.nih.gov/genbank/genomes/Eukaryotes/vertebrates_mammals/Homo_sapiens/GRCh37/PATCHES/"

    #	take a look through the downloaded files to find the name
    #	correspondence.  The fasta names are in:
    zcat patch_release_1/FASTA/alt.scaf.fa.gz | grep "^>" 
    # the other names are in:
    cat patch_release_1/alt_scaffold_placement.txt
    #	Decide on UCSC chrom names
    #	Create a file with these different names to use later:
# fasta string             alt_scaf_name  parent_name  UCSC chrom name
    cat << '_EOF_' > ucscNames.txt
gi|289436847|gb|GL339449.1 HSCHR5_1_CTG1 CM000667.1 chr5_ctg1_hap1
gi|289436846|gb|GL339450.1 HG79_PATCH CM000671.1 chr9_gl339450
chrM_NC_012920 unknown unknown chrM_NC_012920
'_EOF_'
    # << happy emacs


    #	construct the files for UCSC:
    mkdir /hive/data/genomes/hg19/bed/additionalSequence/patch1
    cd /hive/data/genomes/hg19/bed/additionalSequence/patch1

    #	add these sequences to existing hg19.2bit to make a new one:
    cat ../chrM/chrM_NC_012920.fa > patch1.ucsc.fa
    zcat ../patches/patch_release_1/FASTA/alt.scaf.fa.gz \
        | sed -e "s/^>.*GL339449.1.*/>chr5_ctg1_hap1/;" \
-e "s/^>.*GL339450.1.*/>chr9_gl339450/" >> patch1.ucsc.fa

    twoBitToFa /gbdb/hg19/hg19.2bit hg19.existing.fa
    faToTwoBit hg19.existing.fa patch1.ucsc.fa hg19.patch1.2bit
    rm -f /gbdb/hg19/hg19.unmasked.patch1.2bit
    #	temporarily use this unmasked sequence
    ln -s `pwd`/hg19.unmasked.patch1.2bit /gbdb/hg19/hg19.patch1.2bit
    twoBitInfo hg19.patch1.2bit stdout | sort -k2nr > patch1.chrom.sizes

    cat << '_EOF_' > mkTables.pl
#!/usr/bin/env perl

use strict;
use warnings;

sub usage() {
    printf STDERR "usage: mkTables.pl patches.chrom.sizes \\\n";
    printf STDERR " ../patches/ucscNames.txt ../patches/patch_release_1/AGP/alt.scaf.agp.gz\n";
}

my $argc = scalar(@ARGV);

if ($argc < 3) {
    usage;
    exit 255;
}

my $sizes = shift;      # patches.chrom.sizes
my $names = shift;      # patches/ucscNames.txt
my $agpFile = shift;    # alt.scaf.agp.gz

my %glToChr;
my %chrToCtg;
my %fastaToChr;
my %chrToSize;

open(FH, "<$sizes") or die "can not read $sizes";
while (my $line = <FH>) {
    chomp $line;
    my ($chr, $size) = split('\s+', $line);
    $chrToSize{$chr} = $size;
}
close (FH);

open(CI, ">chromInfo.txt") or die "can not write to chromInfo.txt";
open(CT, ">ctgPos.txt") or die "can not write to ctgPos.txt";
open(FH, "<$names");
while (my $line = <FH>) {
    chomp $line;
    my ($faName, $ctg, $cmName, $chr) = split('\s+', $line);
    $faName =~ s/.*gb.GL/GL/;
    my $size = $chrToSize{$chr};
    if (exists($glToChr{$faName})) {
        if ($glToChr{$faName} ne $chr) {
            printf STDERR "ERROR: contig name: $faName was chr name: $glToChr{$faName}\n";
            printf STDERR " now claiming to be chr name: $chr\n";
            exit 255;
        }
    } else {
        $glToChr{$faName} = $chr;
    }
    printf CT "%s\t%d\t%s\t0\t%d\n", $faName, $size, $chr, $size;
    printf CI "%s\t%d\t/gbdb/hg19/hg19.patches.2bit\n", $chr, $size;
}
close (FH);
close (CT);
close (CI);

my $prevObj = "";
my $newIx = 1;
open (GP,">gap.txt") or die "can not write to gap.txt";
open (GL,">gold.txt") or die "can not write to gold.txt";
open (FH,"zcat $agpFile|") or die "can not read $agpFile";
while (my $line = <FH>) {
    next if ($line =~ m/^\s*#/);
    chomp $line;
    my ($object, $objStart, $objEnd, $ix, $type, $frag, $fragStart, $fragEnd, $strand) = split('\s+', $line);
    die "ERROR: can not find contig $object to chr name"
        if (!exists($glToChr{$object}));
    $newIx = 1 if ($prevObj ne $object);
    my $chr = $glToChr{$object};
    if ($type eq "N") {
        # frag is size, fragStart is type of gap, and fragEnd is bridged y/n
        printf GP "%s\t%d\t%d\t%d\t%s\t%d\t%s\t%s\n",
            $chr, $objStart-1, $objEnd, $newIx, $type, $frag, $fragStart,
            $fragEnd;
    } else {
        printf GL "%s\t%d\t%d\t%d\t%s\t%s\t%d\t%d\t%s\n",
            $chr, $objStart-1, $objEnd, $newIx, $type, $frag, $fragStart-1,
            $fragEnd, $strand;
    }
    ++$newIx;
    $prevObj = $object;
    printf "%s\n", $line;
}
close (FH);
close (GL);
close (GP);
'_EOF_'
    # << happy emacs
    chmod +x mkTables.pl
    ./mkTables.pl  patch1.chrom.sizes ../patches/ucscNames.txt \
	../patches/patch_release_1/AGP/alt.scaf.agp.gz
    echo 'chrM_NC_012920  16569   /gbdb/hg19/hg19.patches.2bit' \
	>> chromInfo.txt

    #	create tab files
    hgLoadBed -noLoad -maxChromNameLength=14 \
        -sqlTable=/cluster/home/hiram/kent/src/hg/lib/agpFrag.sql \
        hg19 tGold gold.txt
    rm -f gold.tab
    mv bed.tab gold.tab
    hgLoadBed -noLoad -maxChromNameLength=14 \
        -sqlTable=/cluster/home/hiram/kent/src/hg/lib/gap.sql \
        hg19 tGap gap.txt
    rm -f gap.tab
    mv bed.tab gap.tab

    # these table inserts are performed carefully to make sure they are
    # sane, for example, count the rows before and after load:
    hgsql -e 'load data local infile "gold.tab" into table gold;' hg19
    hgsql -e 'load data local infile "gap.tab" into table gap;' hg19
    hgsql -e 'load data local infile "ctgPos.txt" into table ctgPos;' hg19
    hgsql -e 'load data local infile "chromInfo.txt" into table chromInfo;' hg19

    hgsql -e 'update chromInfo set fileName="/gbdb/hg19/hg19.patch1.2bit";' hg19

    cat << '_EOF_' > ctgPos2.txt
HSCHR5_1_CTG1   1620324 chr5_ctg1_hap1  0       1620324 F
HG79_PATCH      330164  chrUn_gl339450  0       330164  F
NC_012920.1     16569   chrM_NC_012920  0       16569   F
NC_001807.4     16571   chrM    0       16571   O
'_EOF_'
    # << happy emacs

    hgsql -e 'load data local infile "ctgPos2.txt" into table ctgPos2;' hg19

    #	RepeatMasking and SimpleRepeats
    mkdir /hive/data/genomes/hg19/bed/additionalSequence/patch1/RMRun
    cd /hive/data/genomes/hg19/bed/additionalSequence/patch1/RMRun
    ln -s ../patch1.ucsc.fa .
    time /scratch/data/RepeatMasker/RepeatMasker -align -s \
	-species 'Homo sapiens' patch1.ucsc.fa
    # took about 6 hours.  Probably should have broken up the large chr5 bit
    #	sort it:

    head -3 patch1.ucsc.fa.out | sed -e "s/  *$//" > patch1.ucsc.sort.out
    headRest 3 patch1.ucsc.fa.out | sort -k5,5 -k6,6n \
	| sed -e "s/  *$//" >> patch1.ucsc.sort.out
    #	create a .tab file to load
    hgLoadBed -noLoad -maxChromNameLength=14 \
        -sqlTable=$HOME/kent/src/hg/lib/nestedRepeats.sql \
        hg19 tRmsk patch1.ucsc.sort.out
    mv bed.tab patch1.rmsk.tab
    hgsql -e 'load data local infile "patch1.rmsk.tab" into table rmsk;' hg19
    #	create nestedRepeats
    /cluster/bin/scripts/extractNestedRepeats.pl patch1.ucsc.sort.out
    #	create a .tab file to load
    hgLoadBed -noLoad -maxChromNameLength=14 \
        -sqlTable=$HOME/kent/src/hg/lib/nestedRepeats.sql \
        hg19 tNest patch1.nestedRepeats.bed
    rm -f patch1.nestedRepeats.tab
    mv bed.tab patch1.nestedRepeats.tab
    hgsql -e 'load data local infile "patch1.nestedRepeats.tab" into table nestedRepeats;' hg19

    mkdir /hive/data/genomes/hg19/bed/additionalSequence/patch1/simpleRepeat
    cd /hive/data/genomes/hg19/bed/additionalSequence/patch1/simpleRepeat
    ln -s ../patch1.ucsc.fa .
    /cluster/bin/$MACHTYPE/trfBig -trf=/cluster/bin/$MACHTYPE/trf \
	patch1.ucsc.fa /dev/null -bedAt=patch1.ucsc.bed -tempDir=.
    awk '$5 <= 12' patch1.ucsc.bed > trfMask.bed
    mkdir trfMaskChrom
    splitFileByColumn trfMask.bed trfMaskChrom/
    hgLoadBed -oldTable hg19 simpleRepeat patch1.ucsc.bed \
        -sqlTable=$HOME/kent/src/hg/lib/simpleRepeat.sql

    #	add these masks
    cd /hive/data/genomes/hg19/bed/additionalSequence/patch1
    twoBitMask -add hg19.unmasked.patch1.2bit RMRun/patch1.ucsc.sort.out \
	hg19.rmsk.2bit
    ln -s `pwd`/hg19.unmasked.patch1.2bit /gbdb/hg19/hg19.patch1.2bit
    twoBitMask -add hg19.rmsk.2bit simpleRepeat/trfMask.bed hg19.patch1.t.2bit
    #	safe to ignore errors about >= 13 fields
    twoBitToFa hg19.patch1.t.2bit stdout | faSize stdin \
	> hg19.patch1.2bit.faSize 2>&1
    # 3139128321 bases (239950803 N's 2899177518 real 1431272440 upper
    # 1467905078 lower) in 96 sequences in 1 files
    # %46.76 masked total, %50.63 masked real
    #	update the unmasked sequence from earlier:
    rm -f hg19.patch1.2bit; mv hg19.patch1.t.2bit hg19.patch1.2bit

    time blat hg19.patch1.2bit \
	/dev/null /dev/null -tileSize=11 -makeOoc=hg19.patch1.11.ooc \
	-repMatch=1024
    #	Wrote 30723 overused 11-mers to hg19.patch1.11.ooc
    cp -p hg19.patch1.2bit hg19.patch1.11.ooc /hive/data/staging/data/hg19
    mkdir nib
    twoBitToFa -seq=chrM_NC_012920 hg19.patch1.2bit stdout \
	| faToNib -softMask stdin nib/chrM_NC_012920.nib
    twoBitToFa -seq=chr5_ctg1_hap1 hg19.patch1.2bit stdout \
	| faToNib -softMask stdin nib/chr5_ctg1_hap1.nib
    twoBitToFa -seq=chr9_gl339450 hg19.patch1.2bit stdout \
	| faToNib -softMask stdin nib/chr9_gl339450.nib


    mkdir /hive/data/genomes/hg19/bed/additionalSequence/patch1/linSpecRep
    cd /hive/data/genomes/hg19/bed/additionalSequence/patch1/linSpecRep
for C in chr9_gl339450 chrM_NC_012920 chr5_ctg1_hap1
do
    head -3 ../RMRun/patch1.ucsc.sort.out > ${C}.out
    grep "${C} " ../RMRun/patch1.ucsc.sort.out >> ${C}.out
    rm -f ${C}.out_mus*
    /scratch/data/RepeatMasker/DateRepeats ${C}.out -query human -comp mouse
    /cluster/bin/scripts/extractRepeats 1 ${C}.out_mus* > ${C}.out.spec
    rm -f ${C}.out_mus*
done

    # copy new files to /hive/data/staging/data/hg19/ and request rsync
    #	to kluster nodes
-rw-rw-r-- 1      2036 Jul 21 15:39 patch1.chrom.sizes
-rw-rw-r-- 1 816756572 Jul 23 15:32 hg19.patch1.2bit
-rw-rw-r-- 1    122900 Jul 23 15:56 hg19.patch1.11.ooc
-rw-rw-r-- 1      8293 Jul 23 15:59 nib/chrM_NC_012920.nib
-rw-rw-r-- 1    810170 Jul 23 16:01 nib/chr5_ctg1_hap1.nib
-rw-rw-r-- 1    165090 Jul 23 16:01 nib/chr9_gl339450.nib
-rw-rw-r-- 1   34676 Jul 23 16:27 lineageSpecificRepeats/chr9_gl339450.out.spec
-rw-rw-r-- 1     386 Jul 23 16:27 lineageSpecificRepeats/chrM_NC_012920.out.spec
-rw-rw-r-- 1  221381 Jul 23 16:27 lineageSpecificRepeats/chr5_ctg1_hap1.out.spec 
#############################################################################
# Update BlastTab tables for hg19 (Done, Fan, 8/6/2010)
    ssh hgwdev
    mkdir -p /hive/data/genomes/hg19/bed/ucsc.12/hgNearBlastp
    cd /hive/data/genomes/hg19/bed/ucsc.12/hgNearBlastp
    mkdir 100806
    cd 100806
    # Get the proteins used by the other hgNear organisms:
    pepPredToFa hg19 knownGenePep hg19.known.faa
    pepPredToFa mm9 knownGenePep mm9.known.faa
    pepPredToFa rn4 knownGenePep rn4.known.faa
    pepPredToFa danRer6 ensPep danRer6.ensPep.faa
    pepPredToFa dm3 flyBasePep dm3.flyBasePep.faa
    pepPredToFa ce6 sangerPep ce6.sangerPep.faa
    pepPredToFa sacCer2 sgdPep sacCer2.sgdPep.faa

    cat << _EOF_ > config.ra
# Latest human vs. other Gene Sorter orgs:
# mouse, rat, zebrafish, worm, yeast, fly

targetGenesetPrefix known
targetDb hg19
queryDbs mm9 rn4 danRer6 dm3 ce6 sacCer2

hg19Fa /hive/data/genomes/hg19/bed/ucsc.12/ucscGenes.faa
mm9Fa /hive/data/genomes/mm9/bed/ucsc.12/ucscGenes.faa
rn4Fa /hive/data/genomes/rn4/bed/blastp/known.faa
danRer6Fa /hive/data/genomes/danRer6/bed/blastp/danRer6.ensPep.faa
dm3Fa /hive/data/genomes/dm3/bed/flybase5.3/flyBasePep.fa
ce6Fa /hive/data/genomes/ce6/bed/blastp/wormPep190.faa
sacCer2Fa /hive/data/genomes/sacCer2/bed/hgNearBlastp/090218/sgdPep.faa

buildDir /hive/data/genomes/hg19/bed/ucsc.12/hgNearBlastp/100806
scratchDir /hive/data/genomes/hg19/bed/ucsc.12/hgNearBlastp/100806/tmp
_EOF_
doHgNearBlastp.pl -targetOnly config.ra >& do.log & tail -f do.log
#########################################################################
# LIFTOVER TO Hg18 (RE-DONE - 2010-07-26 - Hiram )
    #	preserving the previous 10K liftOver files
    mkdir /hive/data/genomes/hg19/bed/liftOver10K
    cd /hive/data/genomes/hg19/bed/liftOver10K
    ln -s ../blat.hg18.2009-06-04/hg19ToHg18.over.chain.gz .

    #	this liftOver is a 5000 size chunk
    mkdir /hive/data/genomes/hg19/bed/blat.hg18.2010-07-26
    cd /hive/data/genomes/hg19/bed/blat.hg18.2010-07-26
    # -debug run to create run dir, preview scripts...
    #	verifies files can be found
    doSameSpeciesLiftOver.pl -debug hg19 hg18
    # Real run:
    time nice -n +19 doSameSpeciesLiftOver.pl -verbose=2 \
	-bigClusterHub=swarm -dbHost=hgwdev -workhorse=hgwdev \
	 hg19 hg18 > do.log 2>&1
    #	real    115m26.071s

    #	checking liftOver accuracy
    mkdir /hive/data/genomes/hg19/bed/blat.hg18.2010-07-26/refGene
    cd /hive/data/genomes/hg19/bed/blat.hg18.2010-07-26/refGene
    hgsql -N -e "select * from refGene;" hg19 | cut -f2- > refGene.hg19.gp
    wc -l refGene.hg19.gp
    #	36766
    #	the 5K block size lift over chain
    liftOver -genePred refGene.hg19.gp ../hg19ToHg18.over.chain.gz \
	refGene.hg19ToHg18.5K.lift.gp refGene.hg19ToHg18.5K.unMapped.gp
    wc -l refGene.hg19ToHg18.5K.unMapped.gp
    #   830
    #	the 10K block size lift over chain
    liftOver -genePred refGene.hg19.gp \
	../../liftOver10K/hg19ToHg18.over.chain.gz \
	refGene.hg19ToHg18.10K.lift.gp refGene.hg19ToHg18.10K.unMapped.gp
    wc -l refGene.hg19ToHg18.10K.unMapped.gp
    #	820
    #	construct custom track of chain files.
    #	the 5K block size lift over chain
    chainToPsl ../hg19ToHg18.over.chain.gz \
	/hive/data/genomes/hg19/chrom.sizes \
	/hive/data/genomes/hg18/chrom.sizes \
	/hive/data/genomes/hg19/hg19.2bit \
	/hive/data/genomes/hg18/hg18.2bit stdout \
	| pslToBed stdin hg19ToHg18.5K.bed
    #	the 10K block size lift over chain
    chainToPsl ../../liftOver10K/hg19ToHg18.over.chain.gz \
	/hive/data/genomes/hg19/chrom.sizes \
	/hive/data/genomes/hg18/chrom.sizes \
	/hive/data/genomes/hg19/hg19.2bit \
	/hive/data/genomes/hg18/hg18.2bit stdout \
	| pslToBed stdin hg19ToHg18.10K.bed

#############################################################################
# GENSCAN PREDICTIONS (DONE - 2010-07-30 Fan)
# 		      (PARTIALLY RE-DONE AFTER FIXING .LTF FILES - 2010-08-03 Fan)
    #   After several attempts, the old genscan process could not be
    #   successfully completed, nor the new process used by mm9,
    #   even with the manual steps to do gsBig with
    #   smaller windowSize.
    #
    #	A new process is developed to overcome the challenges.
    #   This new build process for hg19 genscan is substantially different from
    #	hg18.  Due to gsBig errors, the chroms are split into 2,000,000 bp
    #   segments and the -windowSize is set to the default value of 1,200,000.
    #   The results are then collected and lifted to original chrom
    #   coordinates and then loaded into 3 genscan tables. 

    ssh hgwdev
    
    mkdir -p /hive/data/genomes/hg19/bed/genscan
    cd /hive/data/genomes/hg19/bed/genscan

    # Check out hg3rdParty/genscanlinux to get latest genscan.
    cvs co hg3rdParty/genscanlinux

# the latest and correct genscan build subdir is newTry

    mkdir -p /hive/data/genomes/hg19/bed/genscan/newTry
    cd /hive/data/genomes/hg19/bed/genscan/newTry

# collect .fa files for all chroms in one subdir

    mkdir -p faOrig
    for f in `cat /hive/data/genomes/hg19/chr.fasta.list`
	do
	cp -p /hive/data/genomes/hg19/$f faOrig
	done

# construct the chrom.list

    ls -1 faOrig |sed -e 's/\.fa//'>chrom.list

# creaet the file template to be used for cluster runs
    cat << '_EOF_' > template
#LOOP
/cluster/bin/x86_64/gsBig {check in line+ $(path1)} {check out line gtf/$(root1).gtf} -trans={check out line pep/$(root1).pep} -subopt={check out line subopt/$(root1).bed} -exe=../../../hg3rdParty/genscanlinux/genscan -par=../../../hg3rdParty/genscanlinux/HumanIso.smat -tmp=/tmp -window=1200000
#ENDLOOP
'_EOF_'

# create the chunk2, lift, and runBlat subdir structures
    
    mkdir -p chunk2
    mkdir -p lift
    mkdir -p runBlat
    
    for f in `cat chrom.list`
	do
	mkdir -p lift/$f
	mkdir -p chunk2/$f
	mkdir -p runBlat/$f/gtf 
	mkdir -p runBlat/$f/pep 
	mkdir -p runBlat/$f/subopt
        cp ./template runBlat/$f
        done

# split the chrom fa files into chunks of 2,000,000 bases or less 
# and create corresponding lift files

    for f in `cat chrom.list`
	do
	cd chunk2/$f
	cat ../../faOrig/$f.fa\
	|faSplit -lift=../../lift/$f/$f.lft gap stdin 2000000 ./ck_
	cd ../..
	done

# LATER FOUND THAT THE SEQUENCE ID IN THE .fa FILES UNDER 
# /hive/data/genomes/hg19/* ARE NOT ALWAYS THE SAME AS THE CHROM ID.
# THIS CAUSED INCORRECT CHROM IDs GENERATED IN THE .lft FILES.
# PERFORM THE FOLLOWING TO FIX THE .lft FILES.

    mkdir fixLift
    cd fixLift

    cat << '_EOF_' > fixAll
fix1 chr6_apd_hap1 apd.chr6.4622290.0.4622290.1
fix1 chr6_cox_hap2 cox.chr6.4795371.0.4795371.1
fix1 chr6_dbb_hap3 dbb.chr6.4610396.0.4610396.1
fix1 chr6_mann_hap4 mann
fix1 chr6_mcf_hap5 mcf.chr6.4833398.0.4833398.1
fix1 chr6_qbl_hap6 qbl.chr6.4611984.0.4611984.1
fix1 chr6_ssto_hap7 ssto
'_EOF_'

    cat << '_EOF_' > fix1
echo
echo
echo processing $1 $2
cat /hive/data/genomes/hg19/bed/genscan/newTry/lift/$1/$1.lft |grep $2
cat /hive/data/genomes/hg19/bed/genscan/newTry/lift/$1/$1.lft |sed -e
"s/${2}/${1}/g" >new/$1.lft
cat new/$1.lft |grep $1
cp new/$1.lft /hive/data/genomes/hg19/bed/genscan/newTry/lift/$1/$1.lft 
'_EOF_'

    chmod +x fix*
    fixAll
    cd ..

# go to memk to run cluster jobs

    ssh memk
    cd /hive/data/genomes/hg19/bed/genscan/newTry

# create genome.list and jobList files for each chrom

    for f in `cat chrom.list`
	do
	cd runBlat/$f
	ls -1 /hive/data/genomes/hg19/bed/genscan/newTry/chunk2/$f/* > genome.list
	gensub2 genome.list single template jobList
	cd ../..
	done

# create batch files

    for f in `cat chrom.list`
	do
	cd runBlat/$f
	para create jobList
	cd ../..
	done

# Send off cluster runs

    for f in `cat chrom.list`
	do
        cd runBlat/$f
	para try
        cd ../..
       done

    for f in `cat chrom.list`
	do
        cd runBlat/$f
	para push
        cd ../..
       done

# When all the cluster runs are finished,
# go back to hgwdev for final data collection and table loading

    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/genscan/newTry

# collect ptf results and lift them and then load into the genscan table 

    mkdir -p gtf
    for f in `cat chrom.list`
	do
	echo
	echo processing $f
	cat runBlat/$f/gtf/*.gtf| liftUp -type=.gtf stdout \
        lift/$f/$f.lft error stdin \
	| sed -e "s/ck_/${f}_/g" > gtf/$f.gtf
        ldHgGene -oldTable hg19 -gtf genscan gtf/$f.gtf
	done

# collect subopt results and lift them and then load them into the
# genscanSubopt table

    mkdir -p subopt
    for f in `cat chrom.list`
	do
        echo
	echo processing $f
	cat runBlat/$f/subopt/*.bed| liftUp -type=.bed stdout \
        lift/$f/$f.lft error stdin \
	| sed -e "s/ck_/${f}_/g" > subopt/$f.bed
	hgLoadBed -oldTable hg19 genscanSubopt subopt/$f.bed
	done

# collect pep results and load them into the genscanPep table

    mkdir -p pep
    rm pep/genscanPep.pep
    for f in `cat chrom.list`
	do
        echo
	echo processing $f
	cat runBlat/$f/pep/*.pep\
	| sed -e "s/ck_/${f}_/g" >> pep/genscanPep.pep
	done
    hgPepPred hg19 generic genscanPep pep/genscanPep.pep

################################################################
# HUMAN FETAL BRAIN EXON ARRAYS (YALE) (DONE 2010-08-03 - Chin)
# Note from Andy: All primary data files were lost, The table
# has all of the original data. 
# The "kent/src/hg/makeDb/hgCgiData/Human/microarrayGroups.ra" file should 
# still be valid.  The hgFixed table sestanBrainAtlasExps should be ok.
# So it's just a matter of getting the hg18.sestanBrainAtlas table in
#  hg19 coordinates.

    mkdir  /hive/data/genomes/hg19/bed/yaleMicroarrays
    cd  /hive/data/genomes/hg19/bed/yaleMicroarrays
    cp  /hive/data/genomes/hg18/bed/yaleMicroarrays/sestanBrainAtlas.bed \
           sestanBrainAtlasHg18.bed
    # In the hg18, thickStart is off by 1, fix it
    # hgsql -e " select thickStart-chromStart from sestanBrainAtlas;" hg18 |
    #   sort | uniq
    # 1
    cat sestanBrainAtlasHg18.bed | awk '{print  $1,$2,$3,$4,$5,$6,$7-1,$8,$9,$10,$11,$12,$13,$14,$15 }' >  sestanBrainAtlasHg18Fixed.bed


    # use liftOver to convert the hg18 coordinates to hg19 directly.
    liftOver sestanBrainAtlasHg18Fixed.bed -bedPlus=8 \
      /hive/data/genomes/hg18/bed/liftOver10K/hg18ToHg19.over.chain.gz \
        sestanBrainAtlasHg19.bed unMapped

    # Check the result of liftOver:
    wc -l sestanBrainAtlasHg1*.bed
    # 877877 sestanBrainAtlasHg18.bed
    # 877469 sestanBrainAtlasHg19.bed
    cat unMapped | awk ' /^chr/ {print $1}' | wc -l       
    408
    cat unMapped | awk ' /^#/|| /^chr/ {print $1, $2, $3,_}' \
       > summaryUnMapped.txt

    # Fix up the result, so that $11(blockSizes) = $8(thickEnd)  - $7
    # (thickStart) without the fix, the checkTableCoors will complain.
     cat sestanBrainAtlasHg19.bed | awk '{print  $1,$2,$3,$4,$5,$6,$7,$8,$9, \
        $10, $8-$7, $12,$13,$14,$15 }' >  sestanBrainAtlasHg19Fixed.bed


    # load the table
    hgLoadBed hg19 sestanBrainAtlas sestanBrainAtlasHg19Fixed.bed
    # Reading sestanBrainAtlasHg19.bed
    # Loaded 877469 elements of size 15
    # Sorted
    # Creating table definition for sestanBrainAtlas
    # Saving bed.tab
    # Loading hg19

    # track stuff done from hg18 days: 
    # kent/src/hg/makeDb/trackDb/human/sestanBrainAtlas.html
    # kent/src/hg/makeDb/trackDb/human/trackDb.ra

#############################################################################
# Updating to patch2 sequence (DONE - 2010-08-18 - Hiram)
#	Most of this business is encapsulated into .sh or .pl scripts in
#	these directories.  They can be used next time with slight alterations.
    mkdir /hive/data/genomes/hg19/bed/additionalSequence/patches/patch_release_2
    cd /hive/data/genomes/hg19/bed/additionalSequence/patches/patch_release_2
    wget --cut-dirs=6 --no-parent --timestamping --no-remove-listing -m \
        -nH --ftp-user=anonymous --ftp-password=yourEmail@your.domain \
"ftp://ftp.ncbi.nlm.nih.gov/genbank/genomes/Eukaryotes/vertebrates_mammals/Homo_sapiens/GRCh37.p2/"

    cd /hive/data/genomes/hg19/bed/additionalSequence/patches
    #	construct a script that can gather the names from the delivered
    #	files and generate UCSC names.  May be useful next time.
    cat << '_EOF_' > gatherNames.pl
#!/usr/bin/env perl

use strict;
use warnings;

sub usage() {
    print STDERR "usage: ./gatherNames.pl patch_release_2\n";
}

my $argc = scalar(@ARGV);

if ($argc != 1) {
    usage;
    exit 255;
}

my $patchDir = shift;

if ( ! -d $patchDir ) {
    print STDERR "ERROR: given directory $patchDir is not a directory or does not exist";
    usage;
    exit 255;
}

my %ctgToChr;
my %ctgToFastaName;
my $fasta = "$patchDir/PATCHES/alt_scaffolds/FASTA/alt.scaf.fa.gz";
open (FH, "zcat $fasta | grep '^>' |")
	or die "ERROR: can not read $fasta";
while ( my $line = <FH> ) {
    chomp $line;
    my ($gi, $rest) = split('\s+',$line,2);
    my ($x, $acc, $y, $gl, $z) = split('\|', $gi);
    my $chr = $rest;
    $chr =~ s/Homo sapiens chromosome //;
    $chr =~ s/ genomic contig.*//;
    $ctgToChr{$gl} = $chr;
    $ctgToFastaName{$gl} = $gi;
    $ctgToFastaName{$gl} =~ s/\|$//;
    $ctgToFastaName{$gl} =~ s/^>//;
#    printf "%s\t%s\n", $gl, $chr;
}
close (FH);

my $placement = "$patchDir/PATCHES/alt_scaffolds/alt_scaffold_placement.txt";
open (FH, "sort -t'\t' -k6,6n $placement|") or die "ERROR: can not read $placement";
while (my $line = <FH>) {
    chomp $line;
    next if ($line =~ m/^\s*#/);
    my ($altAsmName, $primAsmName, $altScafName, $altScafAcc, $parentType,
	$parentName, $parentAcc, $regionName, $ori, $altScafStart,
	$altScafStop, $parentStart, $parentStop, $altStartTail, $altStopTail) =
	split('\t',$line);
    my $chr = $ctgToChr{$altScafAcc};
    die "ERROR: chrom name here does not match: $chr != $parentName"
	if ($chr ne $parentName);
#    printf "chr%s %s\t%s\t%s\t%s\t%s\n", $chr, $parentName, $altScafAcc,
#	$altScafName, $parentAcc, $regionName;
    my $ucscChrName = lc($altScafName);
    if ($ucscChrName =~ m/_patch$/) {
	$ucscChrName = sprintf("chr%d_%s", $parentName, lc($altScafAcc));
	$ucscChrName =~ s/\.1$//;
    } else {
	$ucscChrName =~ s/^hs//;
	$ucscChrName =~ s/_1$//;
	my ($chrNum, $hapNum, $ctgName) = split('_', $ucscChrName, 3);
	$ucscChrName = sprintf("%s_%s_%s", $chrNum, $ctgName, lc($altScafAcc));
	$ucscChrName =~ s/\.1$//;
    }
    printf "%s %s %s %s\n", $ctgToFastaName{$altScafAcc}, $altScafName,
	$parentAcc, $ucscChrName;
}
close (FH);
'_EOF_'
    # << happy emacs
    chmod +x ./gatherNames.pl
    ./gatherNames.pl patch_release_2 > ucscNames.patch2.txt

    # with that name translation file in place, begin to put the sequences
    #	together:
    mkdir /hive/data/genomes/hg19/bed/additionalSequence/patch2
    cd /hive/data/genomes/hg19/bed/additionalSequence/patch2
    cat << '_EOF_' > addSequence.pl
#!/usr/bin/env perl

use strict;
use warnings;

sub usage() {
    print STDERR "usage: ./addSequence.pl ../patches/ucscNames.patch2.txt \\\n";
    print STDERR "\t../patches/patch_release_2/PATCHES/alt_scaffolds/FASTA/alt.scaf.fa.gz\n";
}

my $argc = scalar(@ARGV);

if ($argc != 2) {
    usage;
    exit 255;
}

my %skipSequence;
$skipSequence{"chr9_gl339450"} = 1;
$skipSequence{"chr5_ctg1_gl339449"} = 1;

my $names = shift;
my $fasta = shift;

my %fastaToChrNames;
open(FH, "<$names") or die "ERROR: can not read $names";
while (my $line = <FH>) {
    chomp $line;
    my ($fa, $ctg, $cm, $chr) = split('\s+', $line);
    $fastaToChrNames{$fa} = $chr;
}
close (FH);

open(PA, ">patch2.ucsc.fa") or die "ERROR: can not write to patch2.ucsc.fa";
open(FH, "zcat $fasta|") or die "ERROR: can not zcat $fasta";
my $skipToNext = 0;
while (my $line = <FH>) {
    if ($line =~ m/^>/) {
	my ($fa, $rest) = split('\s+', $line, 2);
	$fa =~ s/\|$//;
	$fa =~ s/^>//;
	die "can not find $fa" if (!exists($fastaToChrNames{$fa}));
	my $chr = $fastaToChrNames{$fa};
	if (exists($skipSequence{$chr})) {
	    $skipToNext = 1;
	} else {
	    printf PA ">%s\n", $chr;
	    $skipToNext = 0;
	}
    } else {
	next if($skipToNext);
	print PA $line;
    }
}
close (FH);
close (PA);

my $here=`pwd`;
chomp $here;
print `twoBitToFa ../patch1/hg19.patch1.2bit hg19.existing.fa`;
print `faToTwoBit hg19.existing.fa patch2.ucsc.fa hg19.patch2.2bit`;
print `rm -f /gbdb/hg19/hg19.patch2.2bit`;
print `ln -s $here/hg19.patch2.2bit /gbdb/hg19`;
print `twoBitInfo hg19.patch2.2bit stdout | sort -k2nr > patch2.chrom.sizes`;
'_EOF_'
    # << happy emacs
    chmod +x addSequence.pl
    ./addSequence.pl ../patches/ucscNames.patch2.txt \
      ../patches/patch_release_2/PATCHES/alt_scaffolds/FASTA/alt.scaf.fa.gz

    sort patch2.chrom.sizes ../patch1/patch1.chrom.sizes | uniq -c \
        | sort -rn | awk '$1 == 1' | awk '{printf "%s\t%s\n", $2, $3}' \
        | sort -k2,2nr > patches.chrom.sizes

    cat << '_EOF_' > mkTables.pl
#!/usr/bin/env perl

use strict;
use warnings;

sub usage() {
    printf STDERR "usage: ./mkTables.pl patches.chrom.sizes \\\n";
    printf STDERR " ../patches/ucscNames.patch2.txt ../patches/patch_release_2/PATCHES/alt_scaffolds/AGP/alt.scaf.agp.gz\n";
}

my $argc = scalar(@ARGV);

if ($argc < 3) {
    usage;
    exit 255;
}

my %skipSequence;
$skipSequence{"GL339449.1"} = 1;
$skipSequence{"GL339450.1"} = 1;

my $sizes = shift;	# patches.chrom.sizes
my $names = shift;	# patches/ucscNames.txt
my $agpFile = shift;	# alt.scaf.agp.gz

my %glToChr;
my %chrToCtg;
my %fastaToChr;
my %chrToSize;

open(FH, "<$sizes") or die "can not read $sizes";
while (my $line = <FH>) {
    chomp $line;
    my ($chr, $size) = split('\s+', $line);
    $chrToSize{$chr} = $size;
}
close (FH);

open(CI, ">chromInfo.txt") or die "can not write to chromInfo.txt";
open(CT, ">ctgPos.txt") or die "can not write to ctgPos.txt";
open(FH, "<$names");
while (my $line = <FH>) {
    chomp $line;
    my ($faName, $ctg, $cmName, $chr) = split('\s+', $line);
    $faName =~ s/.*gb.GL/GL/;
    next if (exists($skipSequence{$faName}));
    my $size = $chrToSize{$chr};
    if (exists($glToChr{$faName})) {
	if ($glToChr{$faName} ne $chr) {
	    printf STDERR "ERROR: contig name: $faName was chr name: $glToChr{$faName}\n";
	    printf STDERR " now claiming to be chr name: $chr\n";
	    exit 255;
	}
    } else {
	$glToChr{$faName} = $chr;
    }
    die "not defined faName" if (!defined($faName));
    die "not defined $faName $chr size" if (!defined($size));
    printf CT "%s\t%d\t%s\t0\t%d\n", $faName, $size, $chr, $size;
    printf CI "%s\t%d\t/gbdb/hg19/hg19.patch2.2bit\n", $chr, $size;
}
close (FH);
close (CT);
close (CI);

my $prevObj = "";
my $newIx = 1;
open (GP,">gap.txt") or die "can not write to gap.txt";
open (GL,">gold.txt") or die "can not write to gold.txt";
open (FH,"zcat $agpFile|") or die "can not read $agpFile";
while (my $line = <FH>) {
    next if ($line =~ m/^\s*#/);
    chomp $line;
    my ($object, $objStart, $objEnd, $ix, $type, $frag, $fragStart, $fragEnd, $strand) = split('\s+', $line);
    next if (exists($skipSequence{$object}));
    die "ERROR: can not find contig $object to chr name"
	if (!exists($glToChr{$object}));
    $newIx = 1 if ($prevObj ne $object);
    my $chr = $glToChr{$object};
    if ($type eq "N") {
	# frag is size, fragStart is type of gap, and fragEnd is bridged y/n
	printf GP "%s\t%d\t%d\t%d\t%s\t%d\t%s\t%s\n",
	    $chr, $objStart-1, $objEnd, $newIx, $type, $frag, $fragStart,
	    $fragEnd;
    } else {
	printf GL "%s\t%d\t%d\t%d\t%s\t%s\t%d\t%d\t%s\n",
	    $chr, $objStart-1, $objEnd, $newIx, $type, $frag, $fragStart-1,
	    $fragEnd, $strand;
    }
    ++$newIx;
    $prevObj = $object;
    printf "%s\n", $line;
}
close (FH);
close (GL);
close (GP);
'_EOF_'
    # << happy emacs
    chmod +x mkTables.pl
    ./mkTables.pl patches.chrom.sizes \
    	../patches/ucscNames.patch2.txt \
	../patches/patch_release_2/PATCHES/alt_scaffolds/AGP/alt.scaf.agp.gz

    hgLoadBed -noLoad -maxChromNameLength=14 \
        -sqlTable=/cluster/home/hiram/kent/src/hg/lib/agpFrag.sql \
        hg19 tGold gold.txt
    rm -f gold.tab
    mv bed.tab gold.tab
    hgLoadBed -noLoad -maxChromNameLength=14 \
        -sqlTable=/cluster/home/hiram/kent/src/hg/lib/gap.sql \
        hg19 tGap gap.txt
    rm -f gap.tab
    mv bed.tab gap.tab

    hgsql -e 'load data local infile "gap.tab" into table gap;' hg19
    hgsql -e 'load data local infile "gold.tab" into table gold;' hg19
    hgsql -e 'load data local infile "ctgPos.txt" into table ctgPos;' hg19

    cat << '_EOF_' > mkCtgPos2.pl
#!/usr/bin/env perl

use strict;
use warnings;

sub usage() {
    print STDERR "usage: ./mkCtgPos2.pl ../patches/ucscNames.patch2.txt \\\n";
    print STDERR "\tpatch2.chrom.sizes\n";
}

my $argc = scalar(@ARGV);

if ($argc != 2) {
    usage;
    exit 255;
}

my %skipSequence;
$skipSequence{"chr9_gl339450"} = 1;
$skipSequence{"chr5_ctg1_gl339449"} = 1;

my $names = shift;
my $sizes = shift;

my %chrSize;
open(FH, "<$sizes") or die "ERROR: can not read $sizes";
while (my $line = <FH>) {
    chomp $line;
    my ($chr, $size) = split('\s+', $line);
    $chrSize{$chr} = $size;
}
close (FH);

open(FH, "<$names") or die "ERROR: can not read $names";
while (my $line = <FH>) {
    chomp $line;
    my ($fa, $ctg, $cm, $chr) = split('\s+', $line);
    next if (exists($skipSequence{$chr}));
    if (exists($chrSize{$chr})) {
	my $size = $chrSize{$chr};
	printf "%s\t%d\t%s\t0\t%d\tF\n", $ctg, $size, $chr, $size;
    }
}
close (FH);
'_EOF_'
    # << happy emacs
    chmod +x mkCtgPos2.pl
    ./mkCtgPos2.pl ../patches/ucscNames.patch2.txt patch2.chrom.sizes \
	> ctgPos2.txt
    hgsql -e 'load data local infile "ctgPos2.txt" into table ctgPos2;' hg19

    cat << '_EOF_' > mkHapLocate.pl
#!/usr/bin/env perl

use strict;
use warnings;

sub usage() {
    print STDERR "usage: ./mkHapLocate.pl ctgPos.txt \\\n";
    print STDERR "\t../patches/patch_release_2/PATCHES/alt_scaffolds/alt_scaffold_placement.txt\n";
}

my $argc = scalar(@ARGV);

if ($argc != 2) {
    usage;
    exit 255;
}

my %skipSequence;
$skipSequence{"chr9_gl339450"} = 1;
$skipSequence{"chr5_ctg1_gl339449"} = 1;

my $ctgPos = shift;
my $placement = shift;

my %ctgToHap;
open(FH, "<$ctgPos") or die "ERROR: can not read $ctgPos";
while (my $line = <FH>) {
    my ($ctg, $size, $hapName, $rest) = split('\s+', $line, 4);
    $ctgToHap{$ctg} = $hapName;
}
close (FH);

open(FH,"<$placement") or die "ERROR: can not read $placement";
while (my $line = <FH>) {
    next if ($line =~ m/^#/);
    chomp $line;
    my ($altAsmName, $primAsmName, $altScafName, $altScafAcc, $parentType,
	$parentName, $parentAcc, $regionName, $ori, $altScafStart,
	$altScafStop, $parentStart, $parentStop, $altStartTail,
	$altStopTail) = split('\t', $line);
    if (exists($ctgToHap{$altScafAcc})) {
	my $hapName = $ctgToHap{$altScafAcc};
	printf "chr%s\t%d\t%d\t%s\n", $parentName, $parentStart-1,
	    $parentStop, $hapName;
    } else {
	print STDERR "not found: $altScafAcc $altScafName\n";
    }
}
close (FH);
'_EOF_'
    # << happy emacs
    chmod +x mkHapLocate.pl
    ./mkHapLocate.pl ctgPos.txt \
../patches/patch_release_2/PATCHES/alt_scaffolds/alt_scaffold_placement.txt \
    > haplotypeLocations.bed
    hgLoadBed -oldTable hg19 haplotypeLocations haplotypeLocations.bed 

    mkdir simpleRepeat
    cd simpleRepeat
    ln -s ../patch2.ucsc.fa
    /cluster/bin/$MACHTYPE/trfBig -trf=/cluster/bin/$MACHTYPE/trf \
      patch2.ucsc.fa /dev/null -bedAt=patch2.ucsc.bed -tempDir=.
    awk '$5 <= 12' patch2.ucsc.bed > trfMask.bed
    splitFileByColumn trfMask.bed trfMaskChrom/
    hgLoadBed -oldTable hg19 simpleRepeat patch2.ucsc.bed \
        -sqlTable=$HOME/kent/src/hg/lib/simpleRepeat.sql

    mkdir ../RMRun
    cd ../RMRun
    ln -s ../patch2.ucsc.fa
    faSplit byname patch2.ucsc.fa ctgs/

    cat << '_EOF_' > runOne
#!/bin/csh -fe

set ctg = $1

set runDir = /hive/data/genomes/hg19/bed/additionalSequence/patch2/RMRun
set src = ${runDir}/ctgs/${ctg}.fa
mkdir -p ${runDir}/out/${ctg}
cd ${runDir}/out/${ctg}

cp -p ${src} .
/scratch/data/RepeatMasker/RepeatMasker -align -s -species 'Homo sapiens' ${ctg}.fa
rm -f ${ctg}.fa
'_EOF_'
    # << happy emacs
    chmod +x runOne
    
    ls ctgs > ctg.list
    cat << '_EOF_' > template
#LOOP
runOne $(root1) {check out line+ out/$(root1)/$(root1).fa.out}
#ENDLOOP
'_EOF_'
    # << happy emacs
    gensub2 ctg.list single template jobList
    ssh swarm
    cd /hive/data/genomes/hg19/bed/additionalSequence/patch2/RMRun
    para create jobList
    mkdir out
    para try
    para push
# Completed: 68 of 68 jobs
# CPU time in finished jobs:      88730s    1478.83m    24.65h    1.03d  0.003 y
# IO & Wait Time:                  6277s     104.62m     1.74h    0.07d  0.000 y
# Average job time:                1397s      23.29m     0.39h    0.02d
# Longest finished job:            4154s      69.23m     1.15h    0.05d
# Submission to last job:          4291s      71.52m     1.19h    0.05d

    find ./out -type f -name "*.fa.out" | head -1 | xargs head -3 \
	> hg19.patch2.out
    find ./out -type f -name "*.fa.out" | xargs -L 1 headRest 3 \
	| sort -k5,5 -k6,6n | sed -e "s/  *$//" >> hg19.patch2.out

    extractNestedRepeats.pl hg19.patch2.out > hg19.nestedRepeats.patch2.bed
    hgLoadBed -noLoad -maxChromNameLength=14 \
        -sqlTable=$HOME/kent/src/hg/lib/nestedRepeats.sql \
        hg19 tNest hg19.nestedRepeats.patch2.bed
    rm -f hg19.nestedRepeats.patch2.tab
    mv bed.tab hg19.nestedRepeats.patch2.tab

    hgLoadOut -tabFile=hg19.patch2.rmsk.tab -nosplit hg19 hg19.patch2.out

    hgsql -e 'load data local infile "hg19.nestedRepeats.patch2.tab" into table nestedRepeats;' hg19
    hgsql -e 'load data local infile "hg19.patch2.rmsk.tab" into table rmsk;' \
	hg19

    mv hg19.patch2.2bit hg19.patch2.0.2bit
    twoBitMask -add hg19.patch2.0.2bit RMRun/hg19.patch2.out hg19.patch2.1.2bit
    twoBitMask -add hg19.patch2.1.2bit \
        simpleRepeat/trfMask.bed hg19.patch2.2bit
    twoBitToFa hg19.patch2.2bit stdout | faSize stdin \
	> faSize.hg19.patch2.2bit.txt
    
#############################################################################
# new blat server for the hg19.patch2 sequence (DONE - 2010-08-18 - Hiram)
    hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
	VALUES ("hg19", "blat4", "17792", "1", "0"); \
	INSERT INTO blatServers (db, host, port, isTrans, canPcr) \
	VALUES ("hg19", "blat4", "17793", "0", "1");' \
	    hgcentraltest

#############################################################################
# establish some liftover chains from reference sequence to new added sequence
#	(WORKING - 2010-08-19 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/additionalSequence/patch2/blat.2010-08-11
    cd /hive/data/genomes/hg19/bed/additionalSequence/patch2/blat.2010-08-11
    twoBitInfo ../hg19.patch2.2bit stdout | sort > patch2.chrom.sizes
    twoBitInfo ../../../../hg19.2bit stdout | sort > hg19.chrom.sizes
    comm -13 hg19.chrom.sizes patch2.chrom.sizes > new.sequence.chrom.sizes
for S in `awk '{print $1}' new.sequence.chrom.sizes`
do
    echo $S
    mkdir -p $S
    twoBitToFa ../hg19.patch2.2bit:${S} ${S}/${S}.fa
done

    faToTwoBit chr*/chr*.fa hg19.patch2.only.2bit
    rm -fr chr*/chr*.fa
    rmdir chr*

    ssh swarm
    cd /hive/data/genomes/hg19/bed/additionalSequence/patch2/blat.2010-08-11
    cat << '_EOF_' > runOne
#!/bin/csh -ef

set hg19 = "/hive/data/genomes/hg19/bed/additionalSequence/patch2/hg19.patch2.2bit"
set runJob = `pwd`/job.csh
set target = $1
set outPsl = $2
set query = `echo $target | sed -e "s/_.*//"`
set qSize = `twoBitInfo ${hg19}:${query} stdout | awk '{print $2}'`
set tSequence = "${hg19}:${target}"
set qSequence = "${hg19}:${query}:0-${qSize}"
mkdir -p psl/${target}
pushd psl/${target}
# echo "${runJob} ${tSequence} ${qSequence} `pwd`/${target}.psl"
${runJob} ${tSequence} ${qSequence} `pwd`/${target}.psl
popd
'_EOF_'
    # << happy emacs
    chmod +x runOne

    cat << '_EOF_' > job.csh
#!/bin/csh -ef

set targetList = $1
set queryListIn = $2
set outPsl = $3

if ($targetList:e == "lst") set targetList = /hive/data/genomes/hg19/bed/blat.hg18.2010-07-26/run.blat/$targetList
if ($queryListIn:e == "lst") set queryListIn = /hive/data/genomes/hg19/bed/blat.hg18.2010-07-26/run.blat/$queryListIn

# Use local disk for output, and move the final result to $outPsl
# when done, to minimize I/O.
set tmpDir = `mktemp -d -p /scratch/tmp doSame.blat.XXXXXX`
pushd $tmpDir

# We might get a .lst or a 2bit spec here -- convert to (list of) 2bit spec:
if ($queryListIn:e == "lst") then
  set specList = `cat $queryListIn`
else
  set specList = $queryListIn
endif

# Further partition the query spec(s) into 5k coord ranges, building up
# a .lst of 2bit specs for blat and a .lft liftUp spec for the results:
cp /dev/null reSplitQuery.lst
cp /dev/null query.lft
foreach spec ($specList)
  set file  = `echo $spec | awk -F: '{print $1;}'`
  set seq   = `echo $spec | awk -F: '{print $2;}'`
  set range = `echo $spec | awk -F: '{print $3;}'`
  set start = `echo $range | awk -F- '{print $1;}'`
  set end   = `echo $range | awk -F- '{print $2;}'`
  if (! -e q.sizes) twoBitInfo $file q.sizes
  set seqSize = `awk '$1 == "'$seq'" {print $2;}' q.sizes`
  set chunkEnd = '0'
  while ($chunkEnd < $end)
    set chunkEnd = `expr $start + 5000`
    if ($chunkEnd > $end) set chunkEnd = $end
    set chunkSize = `expr $chunkEnd - $start`
    echo $file\:$seq\:$start-$chunkEnd >> reSplitQuery.lst
    if (($start == 0) && ($chunkEnd == $seqSize)) then
      echo "$start	$seq	$seqSize	$seq	$seqSize" >> query.lft
    else
      echo "$start	$seq"":$start-$chunkEnd	$chunkSize	$seq	$seqSize" >> query.lft
    endif
    set start = `expr $chunkEnd - 500`
  end
end

# Align unsplit target sequence(s) to .lst of 2bit specs for 5k chunks
# of query:
blat $targetList reSplitQuery.lst tmpUnlifted.psl \
  -tileSize=11 -minScore=100 -minIdentity=98 -fastMap -noHead

# Lift query coords back up:
liftUp -pslQ -nohead tmpOut.psl query.lft warn tmpUnlifted.psl

# Move final result into place:
mv tmpOut.psl $outPsl

popd
rm -rf $tmpDir
'_EOF_'
    # << happy emacs
    chmod +x job.csh

    cat << '_EOF_' > template
#LOOP
runOne $(root1) {check out line+ psl/$(root1)/$(root1).psl}
#ENDLOOP
'_EOF_'
    # << happy emacs

    mkdir psl
    awk '{print $2}' new.sequence.chrom.sizes > target.list
    gensub2 target.list single template jobList
    para create jobList
    para try ... check ... push
    para time
# Completed: 71 of 71 jobs
# CPU time in finished jobs:      29233s     487.22m     8.12h    0.34d  0.001 y
# IO & Wait Time:                  2567s      42.78m     0.71h    0.03d  0.000 y
# Average job time:                 448s       7.46m     0.12h    0.01d
# Longest finished job:            3757s      62.62m     1.04h    0.04d
# Submission to last job:          4070s      67.83m     1.13h    0.05d

    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/additionalSequence/patch2/blat.2010-08-11
for C in `cat target.list`
do
    echo ${C}
    mkdir chain/${C}
    cd chain/${C}
    ln -s ../../psl/${C}/${C}.psl
    axtChain -linearGap=medium -psl ${C}.psl ../../../hg19.patch2.2bit ../../../hg19.patch2.2bit ${C}.chain
    cd ../..
done

    cd chain
    find . -type f | xargs cat | chainSort stdin patch2.sort.chain
    hgLoadChain hg19 patch2Chain patch2.sort.chain
    #	Loading 93897 chains into hg19.patch2Chain

#########################################################################
# Vega gene update (DONE - 2010-08-25 - Hiram)
    #	lookup version number at the Vega WEB site:
    #	http://vega.sanger.ac.uk/index.html
    #	and FTP site:
    #	ftp://ftp.sanger.ac.uk/pub/vega/
    cd /hive/data/genomes/hg19
    #	step wise to verify operation
    doEnsGeneUpdate.pl -vegaGene -ensVersion=39 -stop=download hg19.ensGene.ra
    # they changed their naming convention again.  Look at the FTP site,
    #	fix the download script:
    cd bed/vega.39/download
    ./doDownload.csh
# -rw-rw-r-- 1 12399394 Aug 25 09:15 gtf_file.gz
# -rw-rw-r-- 1  9329992 Aug 25 09:15 Homo_sapiens.VEGA.39.pep.all.fa.gz

    doEnsGeneUpdate.pl -vegaGene -ensVersion=39 \
	-continue=process -stop=process hg19.ensGene.ra
# genePredCheck -db=hg19 vegaPseudo.gp.gz
# checked: 12012 failed: 0
# genePredCheck -db=hg19 not.vegaPseudo.gp.gz
# checked: 103437 failed: 0
# genePredCheck -db=hg19 hg19.allGenes.gp.gz
# checked: 115449 failed: 0

    doEnsGeneUpdate.pl -vegaGene -ensVersion=39 \
	-continue=load -stop=load hg19.ensGene.ra
    featureBits hg19 vegaGene
    # 78097231 bases of 2911519270 (2.682%) in intersection
    featureBits hg19 vegaPseudoGene
    #	8782198 bases of 2911519270 (0.302%) in intersection

#########################################################################
# FOSMID END PAIRS (STARTED 9/1/10 angie, WORKING - 2011-07-07 - Hiram)
    # First I downloaded raw files from NCBI, to see if they are newer
    # than what we have in fosends.3:
    mkdir /hive/data/outside/fosends.4
    cd /hive/data/outside/fosends.4
    wget --timestamping ftp://ftp.ncbi.nih.gov/genomes/FOSMIDS/homo_sapiens/\*
    # The file dates are 2005, but all log files begin with 2002 dates.
    # Good, we can proceed with fosends.3 and won't have to reverse-engineer
    # Terry's lost pipeline for translating the files.
    # take a look at the names
    zcat Hs*.dr.mfa.gz | grep "^>" > sequence.names
    zcat *.trim.log.gz | cut -f6 | sort > clone.names
    # we are going to translate the .T0 .T1 .T2 suffix on the sequence names
    #	to _T0 _T1 _T2 to avoid problems during blat and other parts of
    #	the pipeline
    # using the *.trim.log.gz files, figure out the pairs, write to
    #	files endPairs.txt and singles.txt
    cat << '_EOF_' > pairEnds.pl
#!/bin/env perl

use strict;
use warnings;

my %endIds;	# key is cloneId.subId, value is endId_F,endId_R
open(FH,"zcat Hs.*trim.log.gz|") or die "can not read Hs.*trim.log.gz";
while (my $line = <FH>) {
    next if ($line =~ m/^#/);
    my ($id, $l0, $l1, $endId, $wibr, $cloneId, $fr, $cId, $l2) =
	split('\s+', $line);
    my ($oneEnd, $subId) = split('\.', $endId);
    my $key = "$cloneId.$subId";
    if (exists($endIds{$key})) {
	die "ERROR: third end: $endId $cloneId $endIds{$key}"
		if ($endIds{$key} =~ m/,/);
	if ($fr eq "F") {
	    $endIds{$key} = "$endId,$endIds{$key}";
	} else {
	    $endIds{$key} = "$endIds{$key},$endId";
	}
    } else {
	$endIds{$key} = $endId;
    }
}
close (FH);

open (EP, ">endPairs.txt") or die "can not write to endPairs.txt";
open (SI, ">singles.txt") or die "can not write to singles.txt";
foreach my $key (sort (keys %endIds)) {
    my ($cloneId, $subId) = split('\.', $key);
    if ($endIds{$key} =~ m/,/) {
	my ($fwd, $rev) = split(',', $endIds{$key});
	$fwd =~ s/\./_/;
	$rev =~ s/\./_/;
	printf EP "%s\t%s\t%s\n", $fwd, $rev, $cloneId;
    } else {
	my $fwd = $endIds{$key};
	$fwd =~ s/\./_/;
	printf SI "%s\t%s\n", $fwd, $cloneId;
    }
}
close(EP);
close(SI);
'_EOF_'
    # << happy emacs
    chmod +x pairEnds.pl
    ./pairEnds.pl

    # working in the hg19 build directory
    mkdir /hive/data/genomes/hg19/bed/fosEndPairs
    cd /hive/data/genomes/hg19/bed/fosEndPairs

    mkdir /hive/data/genomes/hg19/bed/fosEndPairs/mapEnds
    cd /hive/data/genomes/hg19/bed/fosEndPairs/mapEnds
    # fixup the names, and upper case all sequence (the names are already uc)
    zcat /hive/data/outside/fosends.4/*.dr.mfa.gz \
	| sed -e "s/^>gnl.* />/; s/\.T/_T/" | tr '[a-z]' '[A-Z]' > fosends.4.fa
    faCount fosends.4.fa > fosends.4.faCount
    tail -1 fosends.4.faCount
# total   1258791003      329912804       274295722       339230145       279976011       35376321        40508700
    # verify nothing broken, nothing lost with the name transition
    faCount /hive/data/outside/fosends.4/*.mfa.gz > /tmp/fs4.faCount
    tail -1 /tmp/fs4.faCount
# total   1258791003      329912804       274295722       339230145       279976011       35376321        40508700

    faSize fosends.4.fa
# 1258791003 bases (35376321 N's 1223414682 real 1223414682 upper 0 lower) in 1615492 sequences in 1 files
# Total size: mean 779.2 sd 190.8 min 12 (G248P84602FB12_T0) max 3784 (G248P89620RG7_T0) median 722

    mkdir splitEnds
    faSplit sequence fosEnds.4.fa 400 splitEnds/fosEnds

    # figure out break points in hg19 around large gaps
    hgsql -N -e "select chrom,chromStart,chromEnd,size from gap;" hg19 \
	| sort -k4nr > hg19.gap.bed
    # script to combine adjacent gaps into single gaps
    cat << '_EOF_' > bedCollapse.pl
#!/bin/env perl

use strict;
use warnings;

my $argc = scalar(@ARGV);

if ($argc < 1) {
    printf STDERR "usage: ./bedCollapse.pl <file.bed>\n";
    printf STDERR "will combine adjacent bed elements into one element\n";
    exit 255
}

my $file = shift;
my $chr = "";
my $prevEnd = 0;
my $start = 0;
my $end = 0;
my $size = 0;
open (FH, "sort -k1,1 -k2,2n $file|") or die "can not read $file";
while (my $line = <FH>) {
    chomp $line;
    my ($c, $s, $e, $rest) = split('\s+', $line, 4);
    $size = $end - $start;
    if (length($chr) > 1) {
        if ($chr ne $c) {
            printf "%s\t%d\t%d\t%d\n", $chr, $start, $end, $size;
            $chr = $c; $start = $s; $end = $e;
        } else {
            if ($s == $end) { 
                $end = $e;
            } else {
                printf "%s\t%d\t%d\t%d\n", $chr, $start, $end, $size;
                $chr = $c; $start = $s; $end = $e;
            }
        }
    } else {
        $chr = $c; $start = $s; $end = $e;
    }
}
printf "%s\t%d\t%d\t%d\n", $chr, $start, $end, $size;
close (FH);
'_EOF_'
    # << happy emacs
    chmod +x bedCollapse.pl
    # filter out for gaps of 50,000 and larger
    ./bedCollapse.pl hg19.gap.bed | awk '$4 > 49999' > hg19.gapBreaks.bed
    cat << '_EOF_' > hg19SplitSpec.pl
#!/bin/env perl

use strict;
use warnings;

my %chromSizes;
open (FH, "<../../../chrom.sizes") or die "can not read ../../../chrom.sizes";
while (my $line = <FH>) {
    chomp $line;
    my ($chr, $size) = split('\s+', $line);
    $chromSizes{$chr} = $size;
}
close (FH);

my %chrDone;	# to measure which chroms have been done
my $curChr = "";
my $start = 0;
my $end = 0;
open (FH, "grep -v '_' hg19.gapBreaks.bed|") or die "can not grep hg19.gapBreaks.bed";
while (my $line = <FH>) {
    chomp $line;
    my ($c, $s, $e, $rest) = split('\s+', $line, 4);
    if (length($curChr) > 0) {
	if ($c eq $curChr) {
	    $end = $s;
	    my $size = $end - $start;
	    die "ERROR: size is zero ? $c $s $e" if ($size < 1);
	    printf "%s\t%d\t%d\t%d\n", $curChr, $start, $end, $size;
	    $chrDone{$curChr} = 1;
	} else {	# finish off previous chrom
	    my $chrSize = $chromSizes{$curChr};
	    if ($start < $chrSize) {
		my $size = $chrSize - $start;
		printf "%s\t%d\t%d\t%d\n", $curChr, $start, $chrSize, $size;
		$chrDone{$curChr} = 1;
	    }
	    $curChr = $c;
	}
    } else {  # first line in the file
	$curChr = $c;
	if ($s > 0) {
	    printf "%s\t0\t%d\t%d\n", $curChr, $s, $s;
	    $chrDone{$curChr} = 1;
	}
    }
    $start = $e;	# next start is this end
    $end = $start;	# next end will be next start or chrSize
}
close(FH);
'_EOF_'
    # << happy emacs
    chmod +x hg19SplitSpec.pl
    ./hg19SplitSpec.pl > hg19.splits.bed

    cd /hive/data/genomes/hg19/bed/fosEndPairs/mapEnds
    # 4M chunks might work a bit better
    ./partitionBed.pl 4000000 10000 hg19.splits.bed > hg19.4M.10K.bed
    mkdir /hive/data/genomes/hg19/bed/fosEndPairs/mapEnds/run4M
    cd /hive/data/genomes/hg19/bed/fosEndPairs/mapEnds/run4M
    awk '{printf "%s:%d-%d\n", $1,$2,$3}' ../hg19.4M.10K.bed > hg19.list
    ls ../../splitEnds | sed -e "s/.fa//" > fosEnds.list

    # XXX this didn't work with -fastMap, too much of the query was knocked
    #	out and the results wouldn't pass the pslReps filter
    #  Need to run blat without any arguments, filter the results
    #	with pslReps
    cat << '_EOF_' > runOne
#!/bin/csh -fe

set t=$1
set q=$2
set c=`echo $t | sed -e 's/:.*//'`
set ctgStart=`echo $t | sed -e 's/.*://; s/-.*//'`
set ctgSize=`echo $t | sed -e 's/:/ /; s/-/ /;' | awk '{printf "%d", $3-$2}'`
set chrSize=`egrep "^$c " /scratch/data/hg19/chrom.sizes | cut -f2`
set result="psl/${t}/${q}.psl"
/bin/mkdir -p "psl/${t}"
set tmpLift="/scratch/tmp/${t}.${q}.lift"
set tResult="/scratch/tmp/${t}.${q}.psl"
echo $ctgStart $t $ctgSize $c $chrSize > ${tmpLift}

blat /scratch/data/hg19/hg19.2bit:${t} ../../splitEnds/${q}.fa stdout \
    | liftUp -type=.psl ${tResult} ${tmpLift} error stdin
pslReps  -nearTop=0.01 -minCover=0.70 -minAli=0.85 -noIntrons ${tResult} \
                        ${result} /dev/null
/bin/rm -f ${tmpLift} ${tResult}
'_EOF_'
    # << happy emacs
    chmod +x runOne

    ssh swarm
    cd /hive/data/genomes/hg19/bed/fosEndPairs/mapEnds/run4M
    gensub2 hg19.list fosEnds.list template jobList
    #	340,470 jobs
    para create jobList
    para try
    para check ... push ... etc
# Completed: 340470 of 340470 jobs
# CPU time in finished jobs:  306516290s 5108604.83m 85143.41h 3547.64d  9.720 y
# IO & Wait Time:              22550855s  375847.59m  6264.13h  261.01d  0.715 y
# Average job time:                 967s      16.11m     0.27h    0.01d
# Longest finished job:           18257s     304.28m     5.07h    0.21d
# Submission to last job:        572370s    9539.50m   158.99h    6.62d

    cd /hive/data/genomes/hg19/bed/fosEndPairs/mapEnds/run4M
    time pslSort dirs raw.psl /scratch/tmp psl/ch* > sort.log 2>&1 &
    # 340470 files in 873 dirs
    # Got 340470 files 583 files per mid file
    #	real    291m28.005s
# -rw-rw-r--   1 11527396213 Jul 13 14:53 raw.psl


    # and now that all those individual results are together, filter all
    # of them
    cd /hive/data/genomes/hg19/bed/fosEndPairs/mapEnds
    time pslReps  -nearTop=0.01 -minCover=0.70 -minAli=0.85 -noIntrons \
	run4M/raw.psl hg19.fosEnds.psl /dev/null
    #	Processed 64042919 alignments
    #	real    11m9.402s
    #	-rw-rw-r-- 1   299504257 Jul 13 15:18 hg19.fosEnds.psl

XXX - running - Wed Jul 13 15:08:08 PDT 2011

    # FYI: to see all minimum covers:
grep "^[0-9]" raw.psl \
	| awk '{printf "%.2f\n", 100*($1+$3)/$11}' | ave stdin
# Q1 7.320000
# median 8.040000
# Q3 11.690000
# average 13.519178
# min 0.980000
# max 100.000000
# count 627281022
# total 8480323865.619440
# standard deviation 13.540966

    cd /hive/data/genomes/hg19/bed/fosEndPairs
    time /cluster/home/hiram/bin/x86_64/pslPairs \
    -tInsert=5000 -minId=0.94 -noBin -min=30000 -max=50000 -slop -short \
	-long -orphan -mismatch -verbose mapEnds/hg19.fosEnds.psl \
	/hive/data/outside/fosends.4/endPairs.txt all_fosends hg19.fosEnds
    #	real    0m10.042s
    # filter for score over 300
    awk '$5 >= 300' hg19.fosEnds.pairs | sort -k1,1 -k2,2n \
	> hg19.fosEndPairs.bed
    wc -l hg19.fosEnds.pairs hg19.fosEndPairs.bed
    #	230791 hg19.fosEnds.pairs
    #	230583 hg19.fosEndPairs.bed

    awk '$5 >= 300' hg19.fosEnds.slop hg19.fosEnds.short hg19.fosEnds.long \
	hg19.fosEnds.mismatch hg19.fosEnds.orphan | sort -k1,1 -k2,2n \
	> hg19.fosEndPairsBad.bed

    # for all names in the hg19.fosEndPairs.bed and hg19.fosEndPairsBad.bed
    # files, extract those names from the mapEnds/hg19.fosEnds.psl file
    # to construct a psl file to load up to represent all ends
    awk '{print $11}' hg19.fosEndPairs.bed hg19.fosEndPairsBad.bed \
	| tr '[,]' '[\n]' | sort -u > hg19.allEnds.names
    # the '\t' here actually needs to be the literal: Ctrl-v i
    #	to get a "real" tab there as the character
    headRest 5 mapEnds/hg19.fosEnds.psl | sort -k10 \
	| join -t '\t' -1 10 -2 1 \
-o 1.1,1.2,1.3,1.4,1.5,1.6,1.7,1.8,1.9,1.10,1.11,1.12,1.13,1.14,1.15,1.16,1.17,1.18,1.19,1.20,1.21 - hg19.allEnds.names | sort -k 14,14 -k 15,15n \
	> hg19.fosEnds.load.psl

    sed -e "s/fosEndPairs/hg19FosEndPairs/" \
	$HOME/kent/src/hg/lib/fosEndPairs.sql > hg19FosEndPairs.sql
    sed -e "s/all_fosends/hg19AllFosEnds/" hg19.fosEndPairs.bed \
	| sort -u | hgLoadBed -notItemRgb hg19 hg19FosEndPairs stdin \
	    -sqlTable=hg19FosEndPairs.sql 
    # Loaded 229708 elements
    # why so few compared to before:
    hgsql -e "select count(*) from fosEndPairs" hg19
    #	384442
    wc -l /hive/data/genomes/hg17/bed/fosends/fosEndPairs.bed
    #	384558 /hive/data/genomes/hg17/bed/fosends/fosEndPairs.bed
    # looking at one of those pairs that did not map:
    # G248P800001RA7 G248P800001FA7
    # compared to /hive/data/outside/fosends.2/fosEnds.fa
    # and /hive/data/outside/fosends.3/fosEnds.fa
    # the end sequences in those old files is shorter than the new ones
    # those shorter sequences more easily pass the minCover filter

    # note - this track isn't pushed to RR, just used for assembly QA
    sed -e "s/fosEndPairsBad/hg19FosEndPairsBad/" \
                 ~/kent/src/hg/lib/fosEndPairsBad.sql > hg19FosEndPairsBad.sql

    sed -e "s/all_fosends/hg19AllFosEnds/" hg19.fosEndPairsBad.bed \
	| sort -u | hgLoadBed -notItemRgb hg19 hg19FosEndPairsBad \
	    hg19.fosEndPairsBad.bed stdin -sqlTable=hg19FosEndPairsBad.sql
    #	Loaded 198665 elements of size 11
    # why do we have so many more ?
    wc -l /hive/data/genomes/hg17/bed/fosends/fosEndPairsBad.bed
    #	30830 /hive/data/genomes/hg17/bed/fosends/fosEndPairsBad.bed

    time hgLoadPsl hg19 -table=hg19AllFosEnds hg19.fosEnds.load.psl
    #	load of hg19AllFosEnds did not go as planned: 1160639 record(s),
    #	0 row(s) skipped, 229 warning(s) loading psl.tab
    #	real    0m35.096s
    #	with some warnings such as:
# Warning 1264 Out of range value adjusted for column 'qBaseInsert' at row 3053
# Warning 1264 Out of range value adjusted for column 'qBaseInsert' at row 10569
# Warning 1264 Out of range value adjusted for column 'qBaseInsert' at row 13642
# ...

# the sequences are already loaded from the hg18 lift:
    grep "^>" /gbdb/hg19/fosends/fosEnds.fromHg18.fa | sed -e 's/>//' \
	| sort > hg18.fosEnds.names
    # but they are not the same names:
    grep "^>" mapEnds/fosends.4.fa | sed -e 's/>//' \
	| sort > hg19.fosEnds.names
    sed -e 's/_T.//' hg19.fosEnds.names | sort -u > hg19.unique.fosEnd.names
    wc -l hg18.fosEnds.names hg19.unique.fosEnd.names
    #	1087670 hg18.fosEnds.names
    #	1567737 hg19.unique.fosEnd.names
    comm -12 hg18.fosEnds.names hg19.unique.fosEnd.names | wc -l
    #	1087670

    grep "^>" /hive/data/outside/fosends.3/fosEnds.fa \
	| sed -e 's/>//' | sort > fosends.3.names
    zcat /hive/data/outside/fosends.4/Hs*.dr.mfa.gz | grep "^>" \
	| awk '{print $2}' | sort > fosends.4.names

    zcat /hive/data/outside/fosends.4/Hs.WGS*.dr.mfa.gz \
       | sed -e "s/^>gnl.* />/; s/\.T/_T/" > fosEnds.4.fa

    mkdir /gbdb/hg19/fosends
    ln -s /hive/data/genomes/hg19/bed/fosEndPairs/mapEnds/fosends.4.fa \
    ln -s /hive/data/genomes/hg19/bed/fosEndPairs/fosEnds.4.fa \
	/gbdb/hg19/fosends/hg19FosEndPairs.fa
    time hgLoadSeq hg19 /gbdb/hg19/fosends/hg19FosEndPairs.fa
    #	1615492 sequences
    #	real    1m26.084s

##############################################################################
# hg18 <-> hg19 difference tracks (DONE - 2010-09-03 - Hiram)
    # this is documented in hg18.txt, same heading as above
    mkdir /hive/data/genomes/hg18/bed/liftOverHg19
    cd /hive/data/genomes/hg18/bed/liftOverHg19

#############################################################################
# HGDP GEOGRAPHIC SNP MAPS (DONE 9/15/10 angie)
    # Project data downloaded and parsed in /hive/data/outside/hgdpGeo,
    # see makeDb/doc/hgdpGeo.txt.
    mkdir /hive/data/genomes/hg19/bed/hgdpGeo
    cd /hive/data/genomes/hg19/bed/hgdpGeo
    # Make an rsId-sorted snp coords file for joining with the hgdpGeo data.
    grep -Fwf /hive/data/outside/hgdpGeo/rsIDs.lst \
      ../snp131/snp131.bed \
    | awk 'BEGIN{OFS="\t";} {print $4, $1, $2, $3, $8;}' \
    | sort > snp131CoordsAndRef.txt
    # How many distinct SNPs in there?  (compare to 657000 from HGDP):
    cut -f 1 snp131CoordsAndRef.txt | uniq | wc -l
#656332
    # Join files to make a track table -- well, first we'll need to 
    # normalize alleles to the + strand:
    join -e ERROR -t'	' -o 1.2,1.3,1.4,1.1,2.2,2.3,2.4,1.5 \
      snp131CoordsAndRef.txt /hive/data/outside/hgdpGeo/hgdpGeoCoordless.txt \
    | sed -re 's/([AGTC])\*/\1/' \
    | sort -k1,1 -k2n,2n \
      > hgdpGeo.fixme
    wc -l hgdpGeo.fixme
#667392 hgdpGeo.fixme
    # Use the snp131 reference allele to detect when we need to rev-comp
    # the alleles to match the + strand.  Also, throw out SNPs for which 
    # the ref allele is multi-base -- it's questionable whether we're giving
    # the right coords (some funny things happen with dbSNP's clustering...):
    cat > fixAlleles.pl <<'_EOF_'
#!/usr/bin/env perl
use warnings;
use strict;
my %rc = ('A' => 'T', 'C' => 'G', 'G' => 'C', 'T' => 'A');
while (<>) {
  chomp;  my ($c, $s, $e, $rs, $ancAl, $derAl, $freqs, $ref) = split;
  next unless ($ref =~ /^[ACGT]$/);
  if ($ancAl ne $ref && $derAl ne $ref) {
    $ancAl = $rc{$ancAl};
    $derAl = $rc{$derAl};
  }
  print join("\t", $c, $s, $e, $rs, $ancAl, $derAl, $freqs) . "\n";
}
'_EOF_'
    # << emacs
    chmod a+x fixAlleles.pl
    ./fixAlleles.pl hgdpGeo.fixme > hgdpGeo.tab
    wc -l hgdpGeo.tab
#667349 hgdpGeo.tab
    hgLoadBed hg19 hgdpGeo hgdpGeo.tab \
      -sqlTable=$HOME/kent/src/hg/lib/hgdpGeo.sql
#Loaded 667349 elements of size 7


###########################################################################
# RECOMBINATION RATES (DONE 2010-08-26 - Chin)

# The STS MArkers track must be completed prior to creating this track

    ssh kkstore02
    cd /hive/data/genomes/hg19/bed
    mkdir -p recombRate
    cd recombRate

# Copy other necessary files here (in future, can take from previous
# version)
# NOTE: these are stable, and could be saved in a permanent spot

    cp -p /projects/hg2/booch/psl/info/decode_all .
    cp -p /projects/hg2/booch/psl/info/marshfield_all .
    cp -p /projects/hg2/booch/psl/info/genethon_all .

# Compared these 3 files with the 3 files of hg17, they are identical.

# Determine maximum concordant set of markers for each of the maps
    /cluster/bin/scripts/assignGPsts -full -maxcon \
        /hive/data/outside/ncbi/sts.11/stsAlias.bed \
        /hive/data/genomes/hg19/bed/sts/stsMarkers_pos.rdb \
        decode_all > decode.marker.rdb
    /cluster/bin/scripts/assignGPsts -full -maxcon \
        /hive/data/outside/ncbi/sts.11/stsAlias.bed \
        /hive/data/genomes/hg19/bed/sts/stsMarkers_pos.rdb \
        marshfield_all > marshfield.marker.rdb
    /cluster/bin/scripts/assignGPsts -full -maxcon \
        /hive/data/outside/ncbi/sts.11/stsAlias.bed \
        /hive/data/genomes/hg19/bed/sts/stsMarkers_pos.rdb \
        genethon_all > genethon.marker.rdb

# Determine the rates for each of the maps
    /cluster/bin/scripts/markers_to_recomb_rate.terry.pl decode.marker.rdb \
            hive/data/genomes/hg19/chrom.sizes 1000000 1000000 \
                > decode_1mb_slide_1mb
    /cluster/bin/scripts/markers_to_recomb_rate.terry.pl genethon.marker.rdb \
            /hive/data/genomes/hg19/chrom.sizes 1000000 1000000 \
                >  genethon_1mb_slide_1mb
# got many "... out of genetic distance order. DISCARDING" messages.

    /cluster/bin/scripts/markers_to_recomb_rate.terry.pl marshfield.marker.rdb \
            /hive/data/genomes/hg19/chrom.sizes 1000000 1000000 \
              > marshfield_1mb_slide_1mb
# Got many "... out of genetic distance order. DISCARDING" messages.

# Convert files to proper format
# which requires the "inserts" file
    # get the size of true chrom:
    cd  /hive/data/genomes/hg19/bed/recombRate  
    cat /hive/data/genomes/hg19/chrom.sizes | awk '$1 !~/_/ && !/^chrM/ \
    {print $1, $2}'  > chr.sizes

    # order contigs on each chrom:
    mkdir /hive/data/genomes/hg19/bed/recombRate/orderedCtg

    cat << '_EOF_' > orderCtg.pl
#!/usr/bin/perl
# create ordered contig lists for each chrom
my $db = hg19;
my $ordDir = "/hive/data/genomes/hg19/bed/recombRate/orderedCtg";
@chroms = (1..22, X, Y);

foreach $chr (@chroms) 
{
my $chrName = "chr$chr";
my $chrFile = "chr$chr.ol";
my $ordFile = "$ordDir/$chrFile";
my $sqlstmt="hgsql $db -s -e \'SELECT c.chromStart, c.contig, c.size, c.chrom, i.size FROM ctgPos c, chromInfo i where c.chrom = \"$chrName\" AND c.chrom=i.chrom order by chromStart ASC\' ";
system(" $sqlstmt  > $ordFile");
}
'_EOF_'
    # << happy emacs


    chmod +x orderCtg.pl
    ./orderCtg.pl

    # create the inserts file 
    cat << '_EOF_' > createInserts.pl
#!/usr/local/bin/perl
# FILE: createInserts
# Author: Terry Furey
# Date: 7/13/2004
# Modified by Chin 2010-10-14
# Description: Create inserts file used in creation of some tracks.
# 	This used to be created when lift files being created

$dir = "/hive/data/genomes/hg19";

@chroms = (1..22, X, Y);

print "#Chrom\tBefore_Contig\tAfter_Contig\tNum_bases\tType\n";
$i = 0;
# Look at certain gaps in chroms
foreach $chr (@chroms) {
    open(AGP, "$dir/$chr/chr$chr.agp");
    while ($line = <AGP>) {
	chomp($line);
	@fields = split("\t", $line);
	$lastctg = "";
	$laststart = 1;
	# Want centromeres, large heterochromatin, short_arms, and large gaps
	if (($fields[6] eq "centromere") ||
	    (($fields[6] eq "heterochromatin") && ($fields[2] - $fields[1] > 1000000)) ||
	    ($fields[6] eq "short_arm") ||
	    (($fields[2] - $fields[1]) > 1000000)) {
	    # Record info about gap
	    $chr[$i] = $fields[0];
	    $start[$i] = $fields[1];
	    $end[$i] = $fields[2];
	    $size[$i] = $end[$i] - $start[$i] + 1;
	    $type[$i] = $fields[6];
	    # Find contigs surrounding gap
            open(ORDCTG, "$dir/bed/recombRate/orderedCtg/chr$chr.ol") || die("Could not open $dir/bed/recombRate/orderedCtg/chr$chr.ol\n") ;
	    # short_arm gaps have no previous contig
	    if ($type[$i] eq "short_arm") {
		$ctg1[$i] = "-";
		$start[$i] = 1;
		# Nest record has next contig
		$line1 = <ORDCTG>;
		@fields1 = split("\t",$line1);
		$ctg2[$i] = $fields1[1];
		# Reset end and recalculate size
		$end[$i] = $fields1[0];
		$size[$i] = $end[$i] - $start[$i] + 1;
	    # non-short_arm gaps
	    } else {
		# Find gap immediately before gap
		while ($line1 = <ORDCTG>) {
		    chomp($line1);
		    @fields1 = split("\t", $line1);
		    # This contig ends where gap begins
		    if (($fields1[0] + $fields1[2] + 1) == $start[$i]) {
			$ctg1[$i] = $fields1[1]; 
			# Succeeding contig is in next record
			if ($line1 = <ORDCTG>) {
			    @fields1 = split("\t", $line1);
			    $ctg2[$i] = $fields1[1];
			    # Reset end coordinate and re-calculate size
			    $end[$i] = $fields1[0];
			    $size[$i] = $end[$i] - $start[$i] + 1;
			} else {
			    $ctg2[$i] = "-";
			}
		    # Keep track of possible previous contigs and starts
		    } elsif (($fields1[0] + $fields1[2] + 1) < $start[$i]) {
			$lastctg = $fields1[1];
			$laststart = $fields1[0] + $fields1[2] + 1;
		    # Another gap separated this gap from previous contig,
		    # so didn't find match and passed it up
		    } elsif (($ctg1[$i] eq "") && ($laststart > 1)) {
			# Set start coordinate to end of last contig
			$ctg1[$i] = $lastctg;
			$start[$i] = $laststart;
			# Reset end coordinate and re-calculate size
			@fields1 = split("\t", $line1);
			$ctg2[$i] = $fields1[1];
			$end[$i] = $fields1[0];
			$size[$i] = $end[$i] - $start[$i] + 1;
		    }
		}
	    }
	    close(ORDCTG); 
	    $i++;
	}
    }
    close(AGP);	
}
$num = $i;

# Print them out
for ($i = 0; $i < $num; $i++) {
    # Don't print out duplicate lines for same gap (i.e. centromere and heterochromatin
    # in same gap
    if (($chr[$i] ne $chr[$i-1]) || (($ctg1[$i] ne $ctg1[$i-1]) && ($start[$i] > $end[$i-1]))) {
	# Large gaps must be heterochromatin
	if ($size[$i] > 3100000) {
	    $type[$i] = "heterochromatin";
	}
	# gaps at beginning must be short_arm
	if ($start[$i] <= 1) {
	    $type[$i] = "short_arm";
	}
	# Only want large heterochromatic regions, not telomeres
	if (($type[$i] ne "heterochromatin") || ($size[$i] > 1000000)) { 
	    print "$chr[$i]\t$ctg1[$i]\t$ctg2[$i]\t$size[$i]\t$type[$i]\n";
	}
    }
}
'_EOF_'
    # << happy emacs
 
    chmod +x createInserts.pl
    ./createInserts.pl > inserts

# Convert files to proper format
    cat << '_EOF_' > convRecombRate.pl
#!/usr/local/bin/perl
# File: convRecombRate
# Author: Terry Furey
# Date: 9/2002
# Modified by Chin 2010-10-14
# Project: Human
# Description: Changes recomb rates in large gaps to nan
# Usage message
if ($#ARGV != 3) {
  print stderr "USAGE: createSetBands <recomb file> <insert file> <ctg dir> <window (kb)>\n";
  exit(1);
}

# Read parameters
$rrfile = shift(@ARGV);
$insfile = shift(@ARGV);
$ctgdir = shift(@ARGV);
$basewind = shift(@ARGV);
$window = $basewind * 1000;

# Determine the golden path positions for each of the inserts
open(INSERT, "<$insfile") || die("Could not open $insfile\n");
$line = <INSERT>; #header
while ($line = <INSERT>) {
  next if (substr($line, 0, 1) eq "#"); 
  chomp($line);
  ($chr, $first, $second, $length, $type) = split(' ',$line);

  $thischr = substr($chr,3);
  open(ORDCTG, "<$ctgdir/orderedCtg/chr$thischr.ol") || die("Could not open $ctgdir/orderedCtg/chr$thischr.ol\n");

  # Determine first window for the insert
  if ($first eq "-") {
    $begin = 0;
    $end = int(($length)/$window)*$window;
  } else {
    $found = 0;
    print stderr "Finding $chr $first\n";
    while(!$found) {
      $line = <ORDCTG>;
      chomp($line);
      ($ctgstart, $ctg, $ctglen, $ctgchr, $chrlen) = split(' ',$line);
      if ($ctg eq $first) {
	$begin = int(($ctgstart + $ctglen)/$window)*$window;
	$end = int(($ctgstart + $ctglen + $length)/$window)*$window + $window;
	$found = 1;
      }
    }
  }
  close(ORDCTG);
  print stderr "$chr $begin - $end $type\n";
  for ($i = $begin; $i < $end; $i=$i+$window) {
    $gap{$chr}{$i} = 1;
  }
}
close(INSERT);

# Now, match up with PCT
open(RR, "<$rrfile") || die("Could not open $rrfile\n");
while ($line = <RR>) {
  chomp($line);
  ($chr, $start, $end, $ave, $female, $male) = split("\t", $line);
  if ($gap{$chr}{$start}) {
    print "$chr\t$start\t$end\tNaN\tNaN\tNaN\n";
  } else {
    print "$line\n";
  }
}
close(RR);
'_EOF_'
    # << happy emacs
    
    chmod +x convRecombRate.pl


./convRecombRate.pl  decode_1mb_slide_1mb inserts \
        . 1000 > decode_1mb_slide_1mb_conv

./convRecombRate.pl  marshfield_1mb_slide_1mb inserts \
        . 1000 > marshfield_1mb_slide_1mb_conv

./convRecombRate.pl  genethon_1mb_slide_1mb inserts \
        . 1000 > genethon_1mb_slide_1mb_conv

# Create bed file and load
    /cluster/bin/scripts/createRRbed decode_1mb_slide_1mb_conv \
        marshfield_1mb_slide_1mb_conv genethon_1mb_slide_1mb_conv \
                > recombRate.bed

    ssh hgwdev
    # reuse the recombRate.sql and recombRate.as from before
    hgLoadBed -noBin -tab \
        -sqlTable=$HOME/kent/src/hg/lib/recombRate.sql \
            hg19 recombRate recombRate.bed


############################################################################
# GENE BOUNDS (RNACLUSTER) (DONE 2010-10-20 - Chin)
# Create rnaCluster table (depends on {est,mrna}OrientInfo)

cd /hive/data/genomes/hg19/bed
mkdir rnaCluster
cd rnaCluster/
mkdir chrom

# Create a list of accessions that come from RAGE libraries and need to
# be excluded.
~/kent/src/hg/geneBounds/clusterRna/generateRageAccList.csh hg19 rage.libs

    cat << '_EOF_' > runClusterRna
#!/bin/csh -fe
foreach f (/hive/data/genomes/hg19/nib/chr*.nib) 
    set c = $f:t:r 
    set out = chrom/$c.bed 
    # Exclude accesions in the RAGE file 
    echo clusterRna -mrnaExclude=hg19.rage.libs hg19 /dev/null $out  -chrom=$c 
    clusterRna -mrnaExclude=hg19.rage.libs -verbose=2 \
    -rna=all_mrna -est=intronEst \ 
    hg19 /dev/null $out -chrom=$c 
 end
'_EOF_'
    # << happy emacs
    chmod +x ./runClusterRna

    ./runClusterRna

hgLoadBed hg19 rnaCluster chrom/*.bed

#############################################################################
# dbSNP BUILD 132 (SNP132) BASIC TABLES (DONE 11/17/10 angie)
# Initially loaded 11/15/10; I found some missing or improperly located rs_fasta
# sequences and dbSNP re-dumped rs_fasta, so I rebuilt everything dependent on
# rs_fasta 11/17.
    # Set up build directory
    mkdir -p /hive/data/outside/dbSNP/132/{human,shared}

    # Get field encodings -- if there are changes or additions to the
    # encoding of the corresponding fields, you might need to update
    # snpNcbiToUcsc, hgTracks, hgc and hgTrackUi (see also
    # hg/lib/snp125Ui.c).
    cd /hive/data/outside/dbSNP/132/shared
    alias wg wget --timestamping
    set ftpShared = ftp://ftp.ncbi.nih.gov/snp/database/shared_data
    wg $ftpShared/LocTypeCode.bcp.gz
    wg $ftpShared/SnpClassCode.bcp.gz
    wg $ftpShared/SnpFunctionCode.bcp.gz
    wg $ftpShared/SnpValidationCode.bcp.gz
    wg $ftpShared/Allele.bcp.gz

    ########################## DOWNLOAD #############################
    cd /hive/data/outside/dbSNP/132/human
    mkdir data schema rs_fasta
    # Get data from NCBI (anonymous FTP)
    set ftpSnpDb = ftp://ftp.ncbi.nih.gov/snp/organisms/human_9606/database
    wg ftp://ftp.ncbi.nih.gov/snp/00readme.txt
    cd /hive/data/outside/dbSNP/132/human/data
    # ContigLoc table has coords, orientation, loc_type, and refNCBI allele
    wg $ftpSnpDb/organism_data/b132_SNPContigLoc_37_1.bcp.gz
    # ContigLocusId table has functional annotations
    wg $ftpSnpDb/organism_data/b132_SNPContigLocusId_37_1.bcp.gz
    wg $ftpSnpDb/organism_data/b132_ContigInfo_37_1.bcp.gz
    # MapInfo has alignment weights
    wg $ftpSnpDb/organism_data/b132_SNPMapInfo_37_1.bcp.gz
    # SNP has univar_id, validation status and heterozygosity
    wg $ftpSnpDb/organism_data/SNP.bcp.gz
    # New info as of 132: allele freq, 'clinical' bit, SNP submitter handles
    wg $ftpSnpDb/organism_data/SNPAlleleFreq.bcp.gz
    wg $ftpSnpDb/organism_data/SNP_bitfield.bcp.gz
    wg $ftpSnpDb/organism_data/Batch.bcp.gz
    wg $ftpSnpDb/organism_data/SubSNP.bcp.gz
    wg $ftpSnpDb/organism_data/SNPSubSNPLink.bcp.gz

    # Get schema
    cd /hive/data/outside/dbSNP/132/human/schema
    wg $ftpSnpDb/organism_schema/human_9606_table.sql.gz
    wg $ftpSnpDb/shared_schema/dbSNP_main_table.sql.gz

    # Get fasta files
    # using headers of fasta files for molType, class, observed
    cd /hive/data/outside/dbSNP/132/human/rs_fasta
    # Re-downloaded 11/17/10 (dbSNP re-dump):
    wg ftp://ftp.ncbi.nih.gov/snp/organisms/human_9606/rs_fasta/\*.gz

    ########################## LOAD NCBI TABLES #############################
    # Simplify names of data files -- strip version & extras to get
    # local canonical table names.
    cd /hive/data/outside/dbSNP/132/human/data
    foreach f (*.bcp.gz)
      set new = `echo $f \
                 | sed -e 's/^b132_SNP//; s/^b132_//; s/_37_1//; s/.bcp//;'`
      mv $f $new
      echo $new
    end

    cd /hive/data/outside/dbSNP/132/human/schema
    zcat human_9606_table.sql.gz \
    | perl -we '$/ = "\nGO\n\n\n"; \
        while (<>) { \
          next unless /^CREATE TABLE \[(b132_(SNP)?)?(ContigInfo|ContigLoc|ContigLocusId|MapInfo|SNP|SNPAlleleFreq)(_37_1)?\]/; \
          s/b132_(SNP)?//; s/_37_1//; \
          s/[\[\]]//g;  s/GO\n\n/;/;  s/smalldatetime/datetime/g; \
          s/ON PRIMARY//g;  s/COLLATE//g;  s/Latin1_General_BIN//g; \
          s/IDENTITY (1, 1) NOT NULL /NOT NULL AUTO_INCREMENT, PRIMARY KEY (id)/g; \
          s/nvarchar/varchar/g;  s/set quoted/--set quoted/g; \
          s/(image|varchar\s+\(\d+\))/BLOB/g; \
          print; \
        }' \
      > table.sql
    zcat dbSNP_main_table.sql.gz \
    | sed -re 's/\r//g;' \
    | perl -we '$/ = "\nGO\n\n\n";  \
        while (<>) { \
          next unless /^CREATE TABLE \[Allele\]/; \
          s/[\[\]]//g;  s/GO\n\n\n/;\n/;  s/smalldatetime/datetime/g; \
          print; \
        }' \
      >> table.sql

    # load on hgwdev
    hgsql -e 'create database hg19snp132'
    cd /hive/data/outside/dbSNP/132/human/schema
    hgsql hg19snp132 < table.sql
    cd ../data

    # Avoid wasting space by excluding mappings to non-reference contigs (ContigInfo.group_label):
    zcat ContigInfo.gz | cut -f 12 | uniq | sort -u
#CRA_TCAGchr7v2
#Celera
#GRCh37
#HuRef
    foreach t (ContigInfo MapInfo ContigLocusId)
      zcat $t.gz \
      | egrep -vw '(Celera|HuRef|CRA_TCAGchr7v2)' \
      | perl -wpe 's/(\d\d:\d\d:\d\d)\.0/$1/g;' \
      | hgLoadSqlTab -oldTable hg19snp132 $t placeholder stdin
    end

    # Compare contig list between liftContigs.lft and reference contigs in ContigInfo.
    cut -f 2 /hive/data/genomes/hg19/jkStuff/liftContigs.lft | sort > /data/tmp/1
    # (HuRef, Celera, CRA_TCAGchr7v2 grepped out above)
    hgsql hg19snp132 -N -B -e 'select contig_acc from ContigInfo;' | sort > /data/tmp/2
    diff /data/tmp/[12]
#1c1
#< NC_001807
#---
#> NC_012920
    # darn mitochondria version oops.  Use NC_012920ToChrM.over.chain generated for snp131.

    # Make sure there are no orient != 0 contigs among those selected.
    hgsql hg19snp132 -NBe \
      'select count(*) from ContigInfo where orient != 0;'
#0

    # ContigLoc is huge, and we want just the reference contig mappings.
    # Keep lines only if they have a word match to some reference contig ID.
    # That probably will allow some false positives from coord matches,
    # but we can clean those up afterward.
    zcat ContigInfo.gz | g -w GRCh37 | cut -f 1 | sort -n > GRCh37ContigInfo.ctg_id.txt
    wc -l GRCh37ContigInfo.ctg_id.txt 
#259 GRCh37ContigInfo.ctg_id.txt
    zcat ContigLoc.gz \
    | grep -Fwf GRCh37ContigInfo.ctg_id.txt \
    | perl -wpe 's/(\d\d:\d\d:\d\d)\.0/$1/g;' \
    | hgLoadSqlTab -oldTable hg19snp132 ContigLoc placeholder stdin
#Warning 1366 Incorrect integer value: '' for column 'loc_sts_uid' at row 1
#Warning 1366 Incorrect integer value: '' for column 'loc_sts_uid' at row 2
#Warning 1366 Incorrect integer value: '' for column 'loc_sts_uid' at row 3
#Warning 1366 Incorrect integer value: '' for column 'loc_sts_uid' at row 4
#...
#load of ContigLoc did not go as planned: 34610577 record(s), 0 row(s) skipped, 4143419 warning(s) loading /dev/stdin
    # Get rid of those false positives (crazy slow, create indices first next time):
    hgsql hg19snp132 -e 'create table ContigLocFix select cl.* from ContigLoc as cl, ContigInfo as ci where cl.ctg_id = ci.ctg_id;'
    hgsql hg19snp132 -e 'drop table ContigLoc; \
                         rename table ContigLocFix to ContigLoc;'

    zcat SNP.gz \
    | perl -wpe 's/(\d\d:\d\d:\d\d)\.0/$1/g;' \
    | hgLoadSqlTab -oldTable hg19snp132 SNP placeholder stdin
#Warning 1366 Incorrect integer value: '' for column 'CpG_code' at row 1
#Warning 1366 Incorrect integer value: '' for column 'map_property' at row 1
#Warning 1366 Incorrect integer value: '' for column 'CpG_code' at row 2
#Warning 1366 Incorrect integer value: '' for column 'map_property' at row 2
#Warning 1265 Data truncated for column 'avg_heterozygosity' at row 3
#Warning 1265 Data truncated for column 'het_se' at row 3
#...
#load of SNP did not go as planned: 30443714 record(s), 0 row(s) skipped, 21366755 warning(s) loading /dev/stdin
    # ... no big deal.
    foreach t (ContigInfo ContigLoc ContigLocusId MapInfo SNP)
     echo -n "${t}:\t"
      hgsql -N -B hg19snp132 -e 'select count(*) from '$t
    end
#ContigInfo:     259
#ContigLoc:      34610479
#ContigLocusId:  19829016
#MapInfo:        30399417
#SNP:    	 30443714

    #################### EXTRACT INFO FROM NCBI TABLES ####################
    # Glom each SNP's function codes together and load up a new hg19Snp132 table.
    # Also extract NCBI's annotations of coding SNPs' effects on translation.
    # We extract ContigLocusId info only for reference assembly mapping.
    # Some SNP's functional annotations are for an alternate assembly, so we will
    # have no NCBI functional annotations to display for those (but our own are 
    # available).
    cd /hive/data/outside/dbSNP/132/human
    # Add indices to tables for a big join (5 or 6 minutes):
    hgsql hg19snp132 -e \
      'alter table ContigInfo add index (ctg_id); \
       alter table ContigLocusId add index (ctg_id);'
    hgsql hg19snp132 -NBe 'select snp_id, ci.contig_acc, asn_from, asn_to, mrna_acc, \
                           fxn_class, reading_frame, allele, residue, codon, cli.ctg_id \
                           from ContigLocusId as cli, ContigInfo as ci \
                           where cli.ctg_id = ci.ctg_id;' \
      > ncbiFuncAnnotations.txt
    wc -l ncbiFuncAnnotations.txt
#19828052 ncbiFuncAnnotations.txt
    # Ignore function code 8 (cds-reference, just means that some allele matches reference)
    # and glom functions for each SNP id:
    cut -f 1-4,6,11 ncbiFuncAnnotations.txt \
    | sort -u -k1n,1n -k6n,6n -k3n,3n -k5n,5n \
    | perl -we 'while (<>) { chomp; \
                  ($id, undef, $s, $e, $f, $c) = split; \
                  if (defined $prevId && $id == $prevId && $c == $prevC && $s == $prevS) { \
                    $prevFunc .= "$f," unless ($f == 8); \
                  } else { \
                    if (defined $prevId) { \
                      print "$prevId\t$prevC\t$prevS\t$prevE\t$prevFunc\n" if ($prevFunc); \
                    } \
                    $prevFunc = ($f == 8) ? "" : "$f,"; \
                  } \
                  ($prevId, $prevC, $prevS, $prevE) = ($id, $c, $s, $e); \
                } \
                print "$prevId\t$prevC\t$prevS\t$prevE\t$prevFunc\n" if ($prevFunc);' \
      > ucscFunc.txt
    wc -l ucscFunc.txt
#11673179 ucscFunc.txt
    cat > ucscFunc.sql <<EOF
CREATE TABLE ucscFunc (
        snp_id int NOT NULL ,
        ctg_id int(10) NOT NULL ,
        asn_from int(10) NOT NULL ,
        asn_to int(10) NOT NULL ,
        fxn_class varchar(255) NOT NULL ,
        INDEX snp_id (snp_id),
        INDEX ctg_id (ctg_id)
);
EOF
    hgLoadSqlTab hg19snp132 ucscFunc{,.sql,.txt}
    # ucscFunc coords are NCBI's 0-based, fully-closed, 2-base-wide insertions.
    # We need to leave the coords alone here so ucscFunc can be joined below.
    # Make a list of SNPs with func anno's that are insertion SNPs, so we can use 
    # the list to determine what type of coord fix to apply to each annotation
    # when making snp130CodingDbSnp below.
    hgsql hg19snp132 -NBe \
      'select ci.contig_acc, cl.asn_from, cl.asn_to, uf.snp_id \
       from ucscFunc as uf, ContigLoc as cl, ContigInfo as ci \
       where uf.snp_id = cl.snp_id and \
             uf.ctg_id = cl.ctg_id and uf.asn_from = cl.asn_from and uf.asn_to = cl.asn_to and \
             cl.loc_type = 3 and \
             cl.ctg_id = ci.ctg_id' \
      > ncbiFuncInsertions.ctg.bed
    wc -l ncbiFuncInsertions.ctg.bed
#1099530 ncbiFuncInsertions.ctg.bed

    # Extract observed allele, molType and snp class from FASTA headers gnl
    mkdir rs_fasta/rejects
    mv rs_fasta/rs_ch{AltOnly,NotOn}.fas.gz rs_fasta/rejects/
    zcat rs_fasta/rs_ch*.fas.gz \
    | grep '^>gnl' \
    | perl -wpe 's/^\S+rs(\d+) .*mol="(\w+)"\|class=(\d+)\|alleles="([^"]+)"\|build.*/$1\t$4\t$2\t$3/ || die "Parse error line $.:\n$_\n\t";' \
    | sort -nu \
      > ucscGnl.txt
#547.545u 92.641s 6:08.04 173.9% 0+0k 0+0io 0pf+0w
    wc -l ucscGnl.txt
#30144822 ucscGnl.txt
# weird -- it shrunk from the original 30152349
    cut -f 1 ucscGnl.txt | uniq | wc -l
#30144822
    cat > ucscGnl.sql <<EOF
CREATE TABLE ucscGnl (
        snp_id int NOT NULL ,
        observed varchar(255) NOT NULL,
        molType varchar(255) NOT NULL,
        class varchar(255) NULL ,
        INDEX snp_id (snp_id)
);
EOF
    hgLoadSqlTab hg19snp132 ucscGnl{,.sql,.txt}

    # Add indices to tables for a big join:
    hgsql hg19snp132 -e \
      'alter table ContigLoc  add index (ctg_id); \
       alter table SNP        add index (snp_id); \
       alter table MapInfo    add index (snp_id);'

#TODO: add SNPAlleleFreq to this query and snpNcbiToUcsc (bitfield/"clinical" too)

    # Big leftie join to bring together all of the columns that we want in snp132,
    # using all of the available joining info:
    hgsql hg19snp132 -NBe \
     'SELECT ci.contig_acc, cl.asn_from, cl.asn_to, cl.snp_id, cl.orientation, cl.allele, \
             ug.observed, ug.molType, ug.class, \
             s.validation_status, s.avg_heterozygosity, s.het_se, \
             uf.fxn_class, cl.loc_type, mi.weight, cl.phys_pos_from \
      FROM \
      ((((ContigLoc as cl JOIN ContigInfo as ci \
               ON cl.ctg_id = ci.ctg_id) \
          LEFT JOIN MapInfo as mi ON mi.snp_id = cl.snp_id and mi.assembly = ci.group_label) \
         LEFT JOIN SNP as s ON s.snp_id = cl.snp_id) \
        LEFT JOIN ucscGnl as ug ON ug.snp_id = cl.snp_id) \
       LEFT JOIN ucscFunc as uf ON uf.snp_id = cl.snp_id and uf.ctg_id = cl.ctg_id \
                                and uf.asn_from = cl.asn_from;' \
      > ucscNcbiSnp.ctg.bed
#73.036u 12.668s 25:32.42 5.5%   0+0k 0+0io 0pf+0w
    wc -l ucscNcbiSnp.ctg.bed 
#34610479 ucscNcbiSnp.ctg.bed

    # Use liftUp for everything except mito, then liftOver for mito.
    # There are some weird cases of length=1 but locType=range... in all the cases 
    # that I checked, the length really seems to be 1 so I'm not sure where they got 
    # the locType=range.  Tweak locType in those cases so we can keep those SNPs:
    grep -vw ^NC_012920 ucscNcbiSnp.ctg.bed \
    | awk -F"\t" 'BEGIN{OFS="\t";} \
           $2 == $3 && $14 == 1 {$14=2; \
                                 if (numTweaked < 10) {print $4 > "/dev/stderr";} \
                                 numTweaked++;}  {print;} \
           END{print numTweaked, "single-base, locType=range, tweaked locType" > "/dev/stderr";}' \
    | liftUp ucscNcbiSnp.bed \
      /hive/data/genomes/hg19/jkStuff/liftContigs.lft warn stdin
#TODO: examine these again, report to dbSNP:
#118203330
#118203339
#118203340
#118203367
#118203389
#118203401
#118203405
#118203425
#118203428
#118203433
#588     single-base, locType=range, tweaked locType
#217.470u 28.776s 2:54.46 141.1% 0+0k 0+0io 0pf+0w
    # For liftOver, convert 0-base fully-closed to 0-based half-open because liftOver
    # doesn't deal with 0-base items.  Fake out phys_pos_from to 0 because many coords
    # will differ, oh well.
    grep -w NC_012920 ucscNcbiSnp.ctg.bed \
    | awk -F"\t" 'BEGIN{OFS="\t";} {$3 += 1; $16 = 0; print;}' \
    | liftOver -bedPlus=3 stdin \
        /hive/data/outside/dbSNP/131/human/NC_012920ToChrM.over.chain stdout chrM.unmapped \
    | awk -F"\t" 'BEGIN{OFS="\t";} {$3 -= 1; print;}' \
    | sort -k2n,2n \
      > chrMNcbiSnp.bed
#2.827u 1.576s 0:48.15 9.1%      0+0k 0+0io 0pf+0w
#2.805u 1.392s 0:51.90 8.0%      0+0k 0+0io 7pf+0w
    cat chrM.unmapped
    # Good, got all but 3 SNPS (rs28693675, rs55749223 and rs112781979, partially deleted/deleted)
    cat chrMNcbiSnp.bed >> ucscNcbiSnp.bed
    wc -l ucscNcbiSnp.bed
#34610476 ucscNcbiSnp.bed

    # Translate NCBI's encoding into UCSC's, and perform a bunch of checks.
    cd /hive/data/outside/dbSNP/132/human/
    # Updated snpNcbiToUCSC for new MAX_SNPID (80M -> 120M), 
    # new named alleles oddball formats: CHLC.GGAA2D04, GDB:190880, SHGC-35515, =D22S272
    # new MAX_SNPSIZE (1k -> 16k)
    snpNcbiToUcsc ucscNcbiSnp.bed /hive/data/genomes/hg19/hg19.2bit snp132
#spaces stripped from observed:
#chr12   6093134 6093134 rs41402545
#count of snps with weight  0 = 69071
#count of snps with weight  1 = 29465124
#count of snps with weight  2 = 523595
#count of snps with weight  3 = 3037908
#count of snps with weight 10 = 1514756
#Skipped 976 snp mappings due to errors -- see snp132Errors.bed
#214.507u 7.542s 4:38.56 79.7%   0+0k 0+0io 0pf+0w
    head snp132Errors.bed
#chr1    11082586        11082587        rs80356737      Unexpected refNCBI "AT" for locType "between" (3) -- expected "-"
#chr1    11082586        11082587        rs80356737      rs80356737 is 1 bases long but refNCBI is different length: AT
#chr1    43392806        43392807        rs80359840      Unexpected refNCBI "CA" for locType "between" (3) -- expected "-"
#chr1    43392806        43392807        rs80359840      rs80359840 is 1 bases long but refNCBI is different length: CA
#chr1    43395420        43395421        rs80359834      Unexpected refNCBI "AC" for locType "between" (3) -- expected "-"
#chr1    43395420        43395421        rs80359834      rs80359834 is 1 bases long but refNCBI is different length: AC
#chr1    43395659        43395660        rs80359833      Unexpected refNCBI "AG" for locType "between" (3) -- expected "-"
#chr1    43395659        43395660        rs80359833      rs80359833 is 1 bases long but refNCBI is different length: AG
#chr1    43396801        43396802        rs80359831      Unexpected refNCBI "GC" for locType "between" (3) -- expected "-"
#chr1    43396801        43396802        rs80359831      rs80359831 is 1 bases long but refNCBI is different length: GC
    wc -l snp*
#  33026121 snp132.bed
#        22 snp132.sql
#       976 snp132Errors.bed
#        18 snp132ExceptionDesc.tab
#   4945948 snp132Exceptions.bed
    # 7M new snps, not a big increase in exceptions (snp131 had 4281351)

    # Make one big fasta file.
    # It's a monster: 18G!  Can we split by hashing rsId?
    zcat rs_fasta/rs_ch*.fas.gz \
    | perl -wpe 's/^>gnl\|dbSNP\|(rs\d+) .*/>$1/ || ! /^>/ || die;' \
      > snp132.fa
#611.912u 114.850s 7:40.41 157.8%        0+0k 0+0io 0pf+0w
    # Check for duplicates.
    grep ^\>rs snp132.fa | sort > /data/tmp/seqHeaders
    wc -l /data/tmp/seqHeaders
#30144822 /data/tmp/seqHeaders
    uniq /data/tmp/seqHeaders | wc -l
#30144822
    # Use hgLoadSeq to generate .tab output for sequence file offsets,
    # and keep only the columns that we need: acc and file_offset.
    # Index it and translate to snpSeq table format.
    hgLoadSeq -test placeholder snp132.fa
#30144822 sequences
#52.698u 14.570s 9:18.88 12.0%   0+0k 0+0io 0pf+0w
    cut -f 2,6 seq.tab > snp132Seq.tab
    rm seq.tab

    # Load up main track tables.
    cd /hive/data/outside/dbSNP/132/human
    hgLoadBed -tab -onServer -tmpDir=/data/tmp -allowStartEqualEnd \
      hg19 snp132 -sqlTable=snp132.sql snp132.bed
#Loaded 33026121 elements of size 17
#124.626u 10.983s 8:20.73 27.0%  0+0k 0+0io 0pf+0w
    hgLoadBed -tab -onServer -tmpDir=/data/tmp -allowStartEqualEnd \
      hg19 snp132Exceptions -sqlTable=$HOME/kent/src/hg/lib/snp125Exceptions.sql -renameSqlTable \
      snp132Exceptions.bed
#Loaded 4945948 elements of size 5
#14.816u 0.930s 0:52.42 30.0%    0+0k 0+0io 0pf+0w
    hgLoadSqlTab hg19 snp132ExceptionDesc ~/kent/src/hg/lib/snp125ExceptionDesc.sql \
      snp132ExceptionDesc.tab
    # Load up sequences.
    mkdir -p /gbdb/hg19/snp
    ln -s /hive/data/outside/dbSNP/132/human/snp132.fa /gbdb/hg19/snp/snp132.fa
    hgLoadSqlTab hg19 snp132Seq ~/kent/src/hg/lib/snpSeq.sql snp132Seq.tab

    # Put in a link where one would expect to find the track build dir...
    ln -s /hive/data/outside/dbSNP/132/human /hive/data/genomes/hg19/bed/snp132

#*** TODO: ask cluster-admin to pack the snp132 table (or whatever tables we'll push)

    # Look at the breakdown of exception categories:
    cd /hive/data/outside/dbSNP/132/human
    cut -f 5 snp132Exceptions.bed | sort | uniq -c | sort -nr
#3644435 MultipleAlignments
# 964493 ObservedMismatch
#  90035 SingleClassTriAllelic
#  77552 SingleClassZeroSpan
#  43631 ObservedTooLong
#  33650 MixedObserved
#  26701 FlankMismatchGenomeShorter
#  25574 SingleClassLongerSpan
#  12222 RefAlleleMismatch
#  11525 DuplicateObserved
#   8340 SingleClassQuadAllelic
#   4463 NamedDeletionZeroSpan
#   2052 FlankMismatchGenomeLonger
#    806 ObservedContainsIupac
#    317 NamedInsertionNonzeroSpan
#    150 FlankMismatchGenomeEqual
#      1 RefAlleleRevComp
#      1 ObservedWrongFormat
#TODO: Sent a few bug reports to dbSNP


############################################################################
# SNP132 ORTHOLOGOUS ALLELES IN CHIMP, ORANG, MACAQUE (DONE 11/22/10 angie)
    mkdir /hive/data/genomes/hg19/bed/snp132Ortho
    cd /hive/data/genomes/hg19/bed/snp132Ortho

    # Following Heather's lead in snp126orthos, filter SNPs to to keep
    # only those with class=single, length=1, chrom!~random;
    # Exclude those with exceptions MultipleAlignments,
    # SingleClassTriAllelic or SingleClassQuadAllelic.
    awk '$5 ~ /^MultipleAlignments|SingleClassTriAllelic|SingleClassQuadAllelic/ {print $4;}' \
      /hive/data/outside/dbSNP/132/human/snp132Exceptions.bed \
    | sort -u \
      > snp132ExcludeIds.txt
    awk '$3-$2 == 1 && $1 !~ /_random/ && $1 !~ /^chrUn/ && $11 == "single" {print;}' \
      /hive/data/outside/dbSNP/132/human/snp132.bed \
    | grep -vFwf snp132ExcludeIds.txt \
      > snp132Simple.bed
#264.984u 13.702s 3:57.29 117.4% 0+0k 0+0io 0pf+0w
    wc -l snp132Simple.bed
#23908516 snp132Simple.bed

    # Glom all human info that we need for the final table onto the
    # name, to sneak it through liftOver: rsId|chr|start|end|obs|ref|strand
    awk 'BEGIN{OFS="\t";} \
        {print $1, $2, $3, \
               $4 "|" $1 "|" $2 "|" $3 "|" $9 "|" $8 "|" $6, \
               0, $6;}' \
      snp132Simple.bed > snp132ForLiftOver.bed
#62.518u 2.141s 1:09.79 92.6%    0+0k 0+0io 0pf+0w

    # Map coords to chimp using liftOver.
    mkdir run.liftOChimp
    cd run.liftOChimp
    mkdir split out
#*** NOTE FOR NEXT TIME: make this 10000 not 50000:
    splitFile ../snp132ForLiftOver.bed 50000 split/chunk
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToPanTro2.over.chain.gz \
        \{check out exists out/panTro2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    ssh swarm
    cd /hive/data/genomes/hg19/bed/snp132Ortho/run.liftOChimp
    para make jobList
#Completed: 479 of 479 jobs
#CPU time in finished jobs:     168182s    2803.03m    46.72h    1.95d  0.005 y
#IO & Wait Time:                 12873s     214.55m     3.58h    0.15d  0.000 y
#Average job time:                 378s       6.30m     0.10h    0.00d
#Longest finished job:            1152s      19.20m     0.32h    0.01d
#Submission to last job:          1165s      19.42m     0.32h    0.01d

    # Map coords to orangutan using liftOver.
    mkdir ../run.liftOPon
    cd ../run.liftOPon
    mkdir out
    ln -s ../run.liftOChimp/split .
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToPonAbe2.over.chain.gz \
        \{check out exists out/ponAbe2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    para make jobList
#Completed: 479 of 479 jobs
#CPU time in finished jobs:     413536s    6892.27m   114.87h    4.79d  0.013 y
#IO & Wait Time:                 29174s     486.23m     8.10h    0.34d  0.001 y
#Average job time:                 924s      15.40m     0.26h    0.01d
#Longest finished job:            2299s      38.32m     0.64h    0.03d
#Submission to last job:          2309s      38.48m     0.64h    0.03d

    # Map coords to macaque using liftOver.
    mkdir ../run.liftOMac
    cd ../run.liftOMac
    mkdir out
    ln -s ../run.liftOChimp/split .
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToRheMac2.over.chain.gz \
        \{check out exists out/rheMac2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    para make jobList
#Completed: 479 of 479 jobs
#CPU time in finished jobs:     463852s    7730.87m   128.85h    5.37d  0.015 y
#IO & Wait Time:                 32857s     547.62m     9.13h    0.38d  0.001 y
#Average job time:                1037s      17.28m     0.29h    0.01d
#Longest finished job:            2354s      39.23m     0.65h    0.03d
#Submission to last job:          2444s      40.73m     0.68h    0.03d

    cd /hive/data/genomes/hg19/bed/snp132Ortho
    # Concatenate the chimp results, sorting by chimp pos in order to
    # efficiently access 2bit sequence in getOrthoSeq.  The output of
    # that is then sorted by the glommed human info field, so that we
    # can use join to combine chimp and macaque results in the next step.
    # Ditto for macaque and orangutan.  Each command pipe takes ~6 minutes:
    sort -k1,1 -k2n,2n run.liftOChimp/out/panTro2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/panTro2/panTro2.2bit \
    | sort > panTro2.orthoGlom.txt
    sort -k1,1 -k2n,2n run.liftOPon/out/ponAbe2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/ponAbe2/ponAbe2.2bit \
    | sort > ponAbe2.orthoGlom.txt
    sort -k1,1 -k2n,2n run.liftOMac/out/rheMac2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/rheMac2/rheMac2.2bit \
    | sort > rheMac2.orthoGlom.txt
    wc -l panTro2.orthoGlom.txt ponAbe2.orthoGlom.txt rheMac2.orthoGlom.txt
#22382184 panTro2.orthoGlom.txt
#21280289 ponAbe2.orthoGlom.txt
#19249238 rheMac2.orthoGlom.txt

    # Use the glommed name field as a key to join up chimp and macaque
    # allele data.  Include glommed name from both files because if only
    # file 2 has a line for the key in 2.1, then 1.1 is empty.  Then plop
    # in the orthoGlom fields from each file, which are in the same order
    # as the chimp and macaque columns of snp132OrthoPanTro2RheMac2.
    join -o '1.1 2.1 1.2 1.3 1.4 1.5 1.6 2.2 2.3 2.4 2.5 2.6' \
      -a 1 -a 2 -e '?' \
      panTro2.orthoGlom.txt ponAbe2.orthoGlom.txt \
    | awk '{if ($1 != "?") { print $1, $3,$4,$5,$6,$7,$8,$9,$10,$11,$12; } \
            else           { print $2, $3,$4,$5,$6,$7,$8,$9,$10,$11,$12; }}' \
      > tmp.txt
#113.229u 24.575s 1:26.02 160.1% 0+0k 0+0io 0pf+0w
    join -o '1.1 2.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 2.2 2.3 2.4 2.5 2.6' \
      -a 1 -a 2 -e '?' \
      tmp.txt rheMac2.orthoGlom.txt \
    | perl -wpe 'chomp; \
        ($glom12, $glom3, $o1Chr, $o1Start, $o1End, $o1Al, $o1Strand, \
         $o2Chr, $o2Start, $o2End, $o2Al, $o2Strand, \
         $o3Chr, $o3Start, $o3End, $o3Al, $o3Strand) = split; \
        $glomKey = ($glom12 ne "?") ? $glom12 : $glom3; \
        ($rsId, $hChr, $hStart, $hEnd, $hObs, $hAl, $hStrand) = \
          split(/\|/, $glomKey); \
        $o1Start =~ s/^\?$/0/;  $o2Start =~ s/^\?$/0/;  $o3Start =~ s/^\?$/0/; \
        $o1End   =~ s/^\?$/0/;  $o2End   =~ s/^\?$/0/;  $o3End   =~ s/^\?$/0/; \
        print join("\t", $hChr, $hStart, $hEnd, $rsId, $hObs, $hAl, $hStrand, \
                         $o1Chr, $o1Start, $o1End, $o1Al, $o1Strand, \
                         $o2Chr, $o2Start, $o2End, $o2Al, $o2Strand, \
                         $o3Chr, $o3Start, $o3End, $o3Al, $o3Strand) . "\n"; \
        s/^.*$//;' \
    | sort -k1,1 -k2n,2n > snp132OrthoPt2Pa2Rm2.bed
#516.332u 101.651s 7:59.63 128.8%        0+0k 0+0io 0pf+0w
    wc -l snp132OrthoPt2Pa2Rm2.bed
#23235355 snp132OrthoPt2Pa2Rm2.bed

    hgLoadBed -tab -onServer -tmpDir=/data/tmp -renameSqlTable \
      -sqlTable=$HOME/kent/src/hg/lib/snpOrthoPanPonRhe.sql \
      hg19 snp132OrthoPt2Pa2Rm2 snp132OrthoPt2Pa2Rm2.bed
#Loaded 23235355 elements of size 22
#87.826u 8.471s 8:22.46 19.1%    0+0k 0+0io 0pf+0w

    # Cleanup:
    rm -r run*/split tmp.txt *.orthoGlom.txt bed.tab
    nice gzip snp132Simple.bed snp132ExcludeIds.txt snp132ForLiftOver.bed &


############################################################################
# DBSNP CODING ANNOTATIONS (132) (DONE 11/17/10 angie)
# These annotations are not restricted to the ones that we display,
# so it wasn't necessary to rebuild this after rebuilding snp132 to 
# include SNPs missing from the first rs_fasta dump.
    cd /hive/data/outside/dbSNP/132/human
    # ncbiFuncAnnotations.txt has NCBI coords: 0-based, fully closed.  
    # For anything except an insertion (0 bases between flanks), 
    # we need to add 1 to the end coord.  For an insertion, we need
    # to add 1 to the start coord.  Make a hash of the insertion IDs,
    # then look up each ID in ncbiFuncAnnotations.txt to tell which
    # transform to apply.
    # Note: sort -u with the keys below is too restrictive -- we need full line uniq.
    perl -we 'open($IDS, "ncbiFuncInsertions.ctg.bed") || die "ids: $!"; \
              while (<$IDS>) { chomp; $ids{$_} = 1; } \
              close($IDS); \
              %coding = (2=>1, 3=>1, 4=>1, 8=>1, 9=>1, 41=>1, 42=>1, 44=>1, 45=>1); \
              while (<>) { \
                chomp;  @w = split("\t"); # id, ctg, start, end, ... \
                next unless $coding{$w[5]}; \
                $bed4 = join("\t", $w[1], $w[2], $w[3], $w[0]); \
                if (exists $ids{$bed4} && $w[3] == $w[2]+1) { \
                  $w[2]++; # 2-base insertions: increment start coord \
                } else { \
                  $w[3]++; # increment end coord to get half-open \
                } \
                print join("\t", @w) . "\n"; \
              }' ncbiFuncAnnotations.txt \
    | sort -k1n,1n -k2,2 -k3n,3n -k5,5 -k6n,6n \
    | uniq \
      > ncbiCodingAnnotations.txt
    wc -l ncbiCodingAnnotations.txt
#1015611 ncbiCodingAnnotations.txt
    # How many & what kinds of function types?
    cut -f 6 ncbiCodingAnnotations.txt \
    | sort -n | uniq -c
# 179089 3   (coding-synon)
# 493143 8   (cds-reference -- ignored)
#  10575 41  (nonsense)
# 272848 42  (missense)
#  57934 44  (frameshift)
#   2022 45  (cds-indel)
    # Gather up multiple annotation lines into one line per {snp, gene, frame}:
    perl -e  'while (<>) { chomp; \
                my ($rsId, $ctg, $s, $e, $txId, $fxn, $frm, $nt, $aa, $codon) = split("\t"); \
                if (defined $lastRs && \
                    ($lastRs != $rsId || $lastCtg ne $ctg || $lastS != $s || \
                     $lastTx ne $txId || $lastFrm ne $frm)) { \
                  if (defined $refRow) { \
                    $fxns = "$refRow->[0],$fxns";  $nts = "$refRow->[1],$nts"; \
                    $aas = "$refRow->[2],$aas";    $codons = "$refRow->[3],$codons"; \
                  } \
                  print "$lastCtg\t$lastS\t$lastE\trs$lastRs\t$lastTx\t$lastFrm\t" . \
                        "$count\t$fxns\t$nts\t$codons\t$aas\n"; \
                  $refRow = undef;  @rows = ();  ($count, $fxns, $nts, $codons, $aas) = (); \
                } \
                ($lastRs, $lastCtg, $lastS, $lastE, $lastTx, $lastFrm) = \
                    ($rsId, $ctg, $s, $e, $txId, $frm); \
                $count++; \
                if ($fxn == 8) { \
                  $refRow = [$fxn, $nt, $aa, $codon]; \
                } else { \
                 $fxns .= "$fxn,";  $nts .= "$nt,";  $aas .= "$aa,";  $codons .= "$codon,"; \
                } \
              } \
              if (defined $refRow) { \
                $fxns = "$refRow->[0],$fxns";  $nts = "$refRow->[1],$nts"; \
                $aas = "$refRow->[2],$aas";    $codons = "$refRow->[3],$codons"; \
              } \
              print "$lastCtg\t$lastS\t$lastE\trs$lastRs\t$lastTx\t$lastFrm\t" . \
                    "$count\t$fxns\t$nts\t$codons\t$aas\n";' \
      ncbiCodingAnnotations.txt \
    | liftUp snp132CodingDbSnp.bed /hive/data/genomes/hg19/jkStuff/liftContigs.lft warn stdin
    hgLoadBed hg19 snp132CodingDbSnp -sqlTable=$HOME/kent/src/hg/lib/snp125Coding.sql \
      -renameSqlTable -tab -notItemRgb -allowStartEqualEnd \
      snp132CodingDbSnp.bed
#Loaded 492412 elements of size 11


############################################################################
# SNPMASKED SEQUENCE FOR SNP132 (DONE 11/22/10 angie)
    mkdir /hive/data/genomes/hg19/snp132Mask
    cd /hive/data/genomes/hg19/snp132Mask

    # Identify rsIds with various problems -- we will exclude those.
    awk '$5 ~ /^MultipleAlignments|ObservedTooLong|ObservedWrongFormat|ObservedMismatch|MixedObserved$/ {print $4;}' \
      /hive/data/outside/dbSNP/132/human/snp132Exceptions.bed \
      | sort -u \
      > snp132ExcludeRsIds.txt
    grep -vFwf snp132ExcludeRsIds.txt \
      /hive/data/outside/dbSNP/132/human/snp132.bed \
      > snp132Cleaned.bed
#262.134u 4.612s 4:39.15 95.5%   0+0k 0+0io 0pf+0w

    # Substitutions:
    mkdir substitutions
    snpMaskSingle snp132Cleaned.bed /hive/data/genomes/hg19/hg19.2bit stdout \
    | faSplit byname stdin substitutions/
#Masked 23848747 snps in 23846674 out of 3134643623 genomic bases
    # 2,361 warnings about differing observed strings at same base position --
    # saved as diffObserved.txt.
#/hive/data/genomes/hg19/hg19.2bit has 3137161264 total bases, but the total number of bases in sequences for which we masked snps is 3134643623 (difference is 2517641)
#53.892u 15.183s 3:24.90 33.7%   0+0k 0+0io 0pf+0w
    # Check that 2517641 is the total #bases in sequences with nothing in snp132Cleaned:
    grep -Fw single snp132Cleaned.bed | cut -f 1 | uniq > /data/tmp/1
    grep -vwf /data/tmp/1 ../chrom.sizes
    grep -vwf /data/tmp/1 ../chrom.sizes \
    | awk 'BEGIN {TOTAL = 0;}  {TOTAL += $2;}  END {printf "%d\n", TOTAL;}'
#2517641
    # Make sure that sizes are identical, first diffs are normal -> IUPAC,
    # and first diffs' case is preserved:
    foreach f (substitutions/chr*.fa)
      faCmp -softMask $f ../[1-9UMXY]*/$f:t |& grep -v "that differ"
    end
#chr1 in substitutions/chr1.fa differs from chr1 at ../1/chr1.fa at base 10326 (y != t)
#chr10 in substitutions/chr10.fa differs from chr10 at ../10/chr10.fa at base 61004 (r != a)
#...
#(output OK -- ambiguous bases replacing [agct] at SNP positions)
    foreach f (substitutions/chr*.fa)
      echo $f:t:r
      mv $f $f:r.subst.fa
      gzip $f:r.subst.fa
    end

    # Insertions & deletions not done.  To date we have only offered substs for download.
    # If there is user demand, use template from snp131 above.

    # Clean up and prepare for download:
    gzip snp132Cleaned.bed &
    foreach d (substitutions)
      pushd $d
        md5sum *.gz > md5sum.txt
        cp /hive/data/genomes/hg19/snp131Mask/$d/README.txt .
      popd
    end
    # Edit the README.txt.

    # Create download links on hgwdev.
    mkdir /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/snp132Mask
    ln -s /hive/data/genomes/hg19/snp132Mask/substitutions/* \
      /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/snp132Mask/

#############################################################################
#CREATE MICROSAT TRACK (DONE 2001-11-13 - Chin)
    ssh hgwdev
    cd /cluster/data/hg19/bed
    mkdir microsat
    cd microsat
    awk '($5==2 || $5==3) && $6 >= 15 && $8 100 && $9 0
    {printf("%s\t%s\t%s\t%dx%s\n", $1, $2, $3, $6, $16);}'
	../simpleRepeat/simpleRepeat.bed > microsat.bed
    hgLoadBed hg19 microsat microsat.bed

#############################################################################
# Add Human RNA-editing track hg18 (Done, galt, 7/12/2010)

# DARNED=DAtabase of RNa EDiting
#http://darned.ucc.ie/
#University College Cork

mkdir -p /hive/data/genomes/hg19/bed/darned
cd /hive/data/genomes/hg19/bed/darned
# create go.csh to download and compose allChroms.bed
./go.csh
hgLoadBed hg19 darned allChroms.bed
# at human, level
# added darned.html
# added trackDb.ra entry

#############################################################################
# lsSnpPdb: import of LS-SNP/PDB data for SNP 131 (2010-12-03 markd)
    # down load from JHU
    ssh genbank
    sudo su - genbank
    cd /cluster/data/genbank
    ./bin/lsSnpPdbDownloadStep hg19
    # load into hgwdev database
    ssh hgwdev
    cd /cluster/data/genbank
    ./bin/lsSnpPdbDbLoadStep hg19
    # once this has been QAed, will auto-update from genbank scripts

#############################################################################
# NEW SNP132 (DONE 3/8/11 angie)
# 3/8/11: Re-ran snpNcbiToUcsc & reloaded to not count PAR SNPs as multiply mapped
# Reloaded 1/24/11 to get rid of a couple exceptions that were derived from dbSNP bitfields
# Previously loaded 1/5/11
    # New table type snp132Ext, with same columns as snp125 plus exceptions,
    # allele freqs, and submitter handles, using new script doDbSnp.pl.
    mkdir -p /hive/data/outside/dbSNP/132/human
    cd /hive/data/outside/dbSNP/132/human
    # Look at the directory listing of ftp://ftp.ncbi.nih.gov/snp/database/organism_data/
    # to find the subdir name to use as orgDir below (human_9606 in this case).
    # Then click into that directory and look for file names like 
    #    b(1[0-9][0-9])_*_([0-9]+_[0-9])
    # -- use the first num for build and the second num_num for buildAssembly.
    # jkStuff/liftContigs.lft maps NCBI contig names to chroms; use that for liftUp.
    cat > config.ra <<EOF
db hg19
orgDir human_9606
build 132
buildAssembly 37_1
liftUp /hive/data/genomes/hg19/jkStuff/liftContigs.lft
EOF
    ~/kent/src/hg/utils/automation/doDbSnp.pl config.ra >& do.log & tail -f do.log
# *** This release contains more than one assembly label.
# *** Please examine this list in case we need to exclude any of these:
#
#CRA_TCAGchr7v2
#Celera
#GRCh37
#HuRef
# *** Add refAssemblyLabel to config.ra.  If keeping all labels, it will
# *** look like this:
#
#refAssemblyLabel CRA_TCAGchr7v2,Celera,GRCh37,HuRef
#
# *** Edit out any of those that are not included in hg19 (e.g. Celera).
# *** Then restart this script with -continue=loadDnSnp .
    # GRCh37 is the only one that corresponds to hg19, so add it to config.ra:
    echo "refAssemblyLabel GRCh37" >> config.ra

    # Try again with updated config:
    ~/kent/src/hg/utils/automation/doDbSnp.pl config.ra \
      -continue=loadDbSnp >>& do.log &
    tail -f do.log

    # 3/8/11:
    cd /hive/data/outside/dbSNP/132/human
    mkdir -p `cat workingDir`
    cp -p ucscNcbiSnp.bed.gz `cat workingDir`
    cd `cat workingDir`
    hgsql hg19 -NBe 'select chrom,chromStart,chromEnd,name from par' > par.bed
    snpNcbiToUcsc -par=par.bed \
      -snp132Ext ucscNcbiSnp.bed.gz /hive/data/genomes/hg19/hg19.2bit snp132
    # No change to snp132Errors.bed; only change to snp132ExceptionDesc.tab was
    # the MultipleAlignments count; snp132.bed items that lost their MultipleAlignments
    # exceptions were X & Y PAR matches.
    gzip snp132.bed snp132ExceptionDesc.tab snp132Errors.bed
    mv snp132* /hive/data/outside/dbSNP/132/human/
    cd /hive/data/outside/dbSNP/132/human/
    # Doh, dbSNP also assigned a weight of 3 to the PAR SNPs, and that is triggering
    # our snp132NonUnique filter (below).  I sent dbSNP an email about that, and will
    # tweak weight to 1 where I find a PAR SNP without MultipleAlignments, since I see
    # it as a bug fix.
    zcat snp132.bed.gz \
    | awk -F"\t" 'BEGIN{OFS="\t";} \
           (($1 == "chrX" && \
             (($3 > 60000 && $2 < 2699520) || ($3 > 154931043 && $2 < 155260560))) || \
            ($1 == "chrY" && \
             (($3 > 10000 && $2 < 2649520) || ($3 > 59034049 && $2 < 59363566)))) && \
           $18 !~/MultipleAlignments/ {$17 = 1;} \
           {print;}' > snp132.parWeightTweak.bed
      wc -l snp132.parWeightTweak.bed
#33026121 snp132.parWeightTweak.bed
    # Make sure only the weight has changed:
    zcat snp132.bed.gz | cut -f 1-16,18-25 > /data/tmp/snp132.weightless.bed
    cut -f 1-16,18-25 snp132.parWeightTweak.bed > /data/tmp/snp132.parTweak.weightless.bed
    cmp /data/tmp/snp132*.weightless.bed
    # No output, good.
    # Reload snp132 with the tweaked weights:
    hgLoadBed -tab -onServer -tmpDir=$TMPDIR -allowStartEqualEnd \
      hg19 snp132 -sqlTable=snp132.sql snp132.parWeightTweak.bed
    zcat snp132ExceptionDesc.tab.gz \
    | hgLoadSqlTab hg19 snp132ExceptionDesc $HOME/kent/src/hg/lib/snp125ExceptionDesc.sql stdin
    gzip snp132.parWeightTweak.bed


#############################################################################
# Agilent arrays (2010-12-01 Andy)
cd /hive/data/genomes/hg19/bed/agilentProbes/
# first move all the lifted versions out of the way
mkdir lifted.2009-07-28/
mv * lifted.2009-07-28/
# FTP download from ftp.agilent.com using given user/pass from Anniek De-witte
# (anniek_de-witte@agilent.com)
# downloaded files are gzipped beds. The files are typically located in a 
# directory called "FOR_UCSC" or something like that.  The user/pass and the
# directory are deleted after it's confirmed they're received, so it's not
# too helpful to mention specifics here.
ftp -u user -p password ftp.agilent.com
> cd directory
> get 014693_D_BED_20100501.bed.gz
> get 014698_D_BED_20100501.bed.gz
> get 014950_D_BED_20100501.bed.gz
> get 021529_D_BED_20100501.bed.gz
> get 021850_D_BED_20100430.bed.gz
> get 021924_D_BED_20100501.bed.gz
> get 022060_D_BED_20100501.bed.gz
> get 023642_D_BED_20100430.bed.gz
> get 028081_D_BED_20101014.bed.gz
> get 029830_D_BED_20100922.bed.gz
# unzip everything
gunzip *
ln -s 022060_D_BED_20100501.bed agilent4x180k.bed
ln -s 021529_D_BED_20100501.bed agilentCgh1x1m.bed
ln -s 014693_D_BED_20100501.bed agilentCgh1x244k.bed
ln -s 014698_D_BED_20100501.bed agilentCgh2x105k.bed
ln -s 021850_D_BED_20100430.bed agilentCgh2x400k.bed
ln -s 014950_D_BED_20100501.bed agilentCgh4x44k.bed
ln -s 021924_D_BED_20100501.bed agilentCgh8x60k.bed
ln -s 028081_D_BED_20101014.bed agilentCghSnp2x400k.bed
ln -s 029830_D_BED_20100922.bed agilentCghSnp4x180k.bed
ln -s 023642_D_BED_20100430.bed agilentHrd1x1m.bed
for bed in agilent*.bed; do
    tail -n +2 $bed | hgLoadBed hg19 ${bed%.bed} stdin
done
rm bed.tab
### Update (2011-11-01 Andy Pohl
# (acquired bed file in e-mail attachment)
cd /hive/data/genomes/hg19/bed/agilentProbes
ln -s 030587_D_BED_20101006_Colored.bed agilentCghSnpCancer4x180k.bed
tail +3 agilentCghSnpCancer4x180k.bed | hgLoadBed hg19 agilentCghSnpCancer4x180k stdin


#################################################################################
# Rfam (2011-11-30 Melissa Cline)
#
# This contains genomic alignments of Rfam sequences, from the Rfam group.
#
# This data is used in building UCSC Genes.
#
cd /hive/data/outside/Rfam
mkdir 111130
cd 111130
wget ftp://ftp.sanger.ac.uk/pub/databases/Rfam/CURRENT/genome.gff3.tar.gz
tar xzvf genome.gff3.tar.gz
mkdir hg19
cat /hive/data/genomes/hg19/chrom.aliases \
 |awk '{ print("cat /hive/data/outside/Rfam/111130/genome_gff/" $1 ".gff3",  
               "|sed", sprintf("%c", 39) "s/" $1 "/" $2 "/" sprintf("%c", 39))}' |bash \
 |awk '{ print($1 "\t" $4 - 1 "\t" $5 "\t" $9 "\t1\t" $7 "\t" 
               $4 - 1 "\t" $5 "\t0\t1\t" $5 - $4 + 1 "\t0"  }' \
> hg19/Rfam.bed      




#####################################################
# Vista Enhancers (galt 2010-12-09 done)
#
# Vista from Lawrence-Berkeley has assayed
# 301 human conserved non-coding intra- and inter-
# genic elements for their ability to promote
# lacZ in mouse embryos.  A positive looks like
# a mouse with a purple spine.
#

mkdir /hive/data/genomes/hg19/bed/vistaEnhancers
cd /hive/data/genomes/hg19/bed/vistaEnhancers

# download data file from the vista browser (coordinates are for hg19)
wget -O enhancerbrowser.datadownload.txt 'http://enhancer.lbl.gov/cgi-bin/imagedb3.pl?page_size=100;show=1;search.result=yes;form=search;search.form=no;action=search;search.sequence=1'

# give elements with positive label a score of 900, 
# give elements with negative label a score of 200.
# print to 5-field bed file
cat enhancerbrowser.datadownload.txt \
        | grep ">" \
        | sed -e 's#^<pre>##' \
        | sed -e 's#</pre>$##' \
        | grep "^>Human" \
        | sed -e 's#^>Human|##' \
        | tr :- ' ' \
        | sed -e 's/positive/900/'\
        | sed -e 's/negative/200/' \
        | awk '{print $1"\t"$2"\t"$3"\telement_"$6"\t"$8}' \
        | grep -P -v "^chr\t" \  
        > vistaEnhancers.bed
hgLoadBed hg19 vistaEnhancers vistaEnhancers.bed
#Loaded 1339 elements of size 5


# add to hg19/trackDb.ra
track vistaEnhancers override
url http://enhancer.lbl.gov/cgi-bin/imagedb3.pl?form=presentation&show=1&experiment_id=$$&organism_id=1


#####################################################
# UNIGENE/SAGE TRACK (RE-BUILT - 2010-12-10 Fan)

# Create the uniGene alignments

    # Download of the latest UniGene version is now automated by a
    # cron job -- see /cluster/home/angie/crontab ,
    # /cluster/home/angie/unigeneVers/unigene.csh .

    ssh hgwdev
    mkdir -p /hive/data/genomes/hg19/bed/uniGene/101210
    cd /hive/data/genomes/hg19/bed/uniGene/101210

    set Version = 228

    zcat /hive/data/outside/uniGene/uniGene.$Version/Hs.seq.uniq.gz|\
    sed -e "s#>.*/ug=#>#; s# /len.*##;" > Hs.seq.uniq.simpleHeader.fa

    ssh swarm
    set Version = 228
    mkdir -p /hive/data/genomes/hg19/bed/uniGene/101210/run.blat
    cd /hive/data/genomes/hg19/bed/uniGene/101210/run.blat

    ls -1 /hive/data/genomes/hg19/nib/*.nib > genome.lst
    ls -1S \
    /hive/data/genomes/hg19/bed/uniGene/101210/Hs.seq.uniq.simpleHeader.fa \
      > uniGene.lst
    
    cat << '_EOF_' > template.sub
#LOOP
/cluster/bin/x86_64/blat -repeats=lower -minIdentity=95 ooc=/hive/data/genomes/hg19/11.ooc $(path1) $(path2)  {check out line+ psl/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'

    gensub2 genome.lst uniGene.lst template.sub para.spec
    para create para.spec
    mkdir psl
    para try
    para check
    para push

#Completed: 93 of 93 jobs
#CPU time in finished jobs:      68896s    1148.26m    19.14h    0.80d  0.002 y
#IO & Wait Time:                  4789s      79.82m     1.33h    0.06d  0.000 y
#Average job time:                 792s      13.21m     0.22h    0.01d
#Longest finished job:            5274s      87.90m     1.47h    0.06d
#Submission to last job:          5840s      97.33m     1.62h    0.07d
#Estimated complete:                 0s       0.00m     0.00h    0.00d

    pslSort dirs raw.psl tmp psl >& pslSort.log
    cat raw.psl|\
    pslReps -minCover=0.2 -sizeMatters -minAli=0.965 -nearTop=0.002 \
      stdin hg19.uniGene.pslReps.psl /dev/null
    
    gzip raw.psl

    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/uniGene/101210/run.blat

    hgLoadPsl -table=uniGene_3 hg19 hg19.uniGene.pslReps.psl

    mkdir -p /gbdb/hg19/uniGene
    cd /gbdb/hg19/uniGene

    rm Hs.seq.uniq.simpleHeader.fa
    ln -s \
    /hive/data/genomes/hg19/bed/uniGene/101210/Hs.seq.uniq.simpleHeader.fa \
    Hs.seq.uniq.simpleHeader.fa

# load the sequence

    hgLoadSeq -replace hg19 /gbdb/hg19/uniGene/Hs.seq.uniq.simpleHeader.fa

##############################################################################

##############################################################################
# GAD View Lift (DONE, Andy 2010-12-12)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir gad
cd gad/
echo "select * from gad" | hgsql hg18 | tail -n +2 > hg18.bed
liftOver hg18.bed /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.{bed,unmapped}
hgLoadBed -noBin hg19 gad hg19.bed
rm bed.tab
wc -l *.bed
#  1883 hg18.bed
#  1860 hg19.bed
grep -i split hg19.unmapped | wc -l
#18
grep -i part hg19.unmapped | wc -l
#5
for table in gadAll gadList; do 
    hgsqldump hg18 $table | hgsql hg19
done

#############################################################################
# EVOFOLD (Done, 2010-12-13) - Galt using Jakob's procedure from hg18.txt.

# RNA secondary structure predictions lifted from hg17 and filtered
  ssh -C hgwdev
  mkdir -p /cluster/data/hg19/bed/evofold
  cd /cluster/data/hg19/bed/evofold
  echo "select chrom, chromStart, chromEnd, name, score, strand, size, secStr, conf from evofold;" | hgsql hg17 | sed -e 1d > foldsHg17.bed
  liftOver -bedPlus=6 -minMatch=1.0 foldsHg17.bed /cluster/data/hg17/bed/liftOver/hg17ToHg19.over.chain.gz tmp.bed unmapped.bed
  # remove elements which are wrong size after lifting
  awk '$3-$2 == $7' tmp.bed | sort -k4,4 > rawFoldsHg19.bed

  # structure filters
  # first, remove pairs that can't form in human
  cut -f 1-6 rawFoldsHg19.bed > tmp.bed
  # sequenceForBed can be found and compiled from here: $HOME/kent/src/hg/altSplice/altSplice/
  nice sequenceForBed -db=hg19 -bedIn=tmp.bed -fastaOut=tmp.fa
  cat tmp.fa | sed -e 's/\.[+-]\.chr.*$//' \
             | sed -e '/^>/s/$/\t/' | tr -d '\n' | sed -e 's/>/\n/g' | sed -e '1d' -e '$s/$/\n/' | sort -k1,1 > foldsHg19Seq.tab
  # Several python scripts were originally in /cluster/home/jsp/scripts/
  # I copied them to this directory and
  # I removed the optional "psyco" speedup library which does not work with our 64-bit python
  join -1 4 -2 1 -o "1.4 1.8 2.2" rawFoldsHg19.bed foldsHg19Seq.tab | sed -e 's/  */\t/g' | sort -k1,1 \
	     | ./tabFoldFilter.py > cleanFolds.tab
  join -1 4 -2 1 -o "1.1 1.2 1.3 1.4 1.5 1.6 1.7 2.2 1.9" rawFoldsHg19.bed cleanFolds.tab | sed -e 's/  */\t/g' > tmp1.bed
  # second, remove poor predictions
  # scripts can be found in cvs tree at: cvsroot/jsp/scripts/. They use a few modules which can be found at: cvsroot/jsp/py_modules
  cat tmp1.bed | ./bedRnassFilter.py --dangling --minAvrStemSize=3 | ./bedRnassFilter.sh 1 3 \
	       | ./roundListFloats.py -c9 > foldsHg19.bed
  # clean up
  rm tmp.bed tmp1.bed foldsHg17.bed foldsHg19Seq.tab rawFoldsHg19.bed tmp.fa cleanFolds.tab

  # upload
  hgLoadBed -notItemRgb -sqlTable=$HOME/kent/src/hg/lib/evofold.sql hg19 evofold foldsHg19.bed
#############################################################################
# CREATE .PNG PICTURE FILES OF EVOFOLD RNA STRUCTURES. (DONE, 4/29/2011, Fan)

ssh hgwdev
mkdir /hive/data/genomes/hg19/bed/evofold/doEvoFold
cd /hive/data/genomes/hg19/bed/evofold/doEvoFold

# Creaet sub-directories to store .png files (total of about 47.5 K of them) sparated by chromosomes.

mkdir -p evoFold/chr1
mkdir -p evoFold/chr10
mkdir -p evoFold/chr11
mkdir -p evoFold/chr12
mkdir -p evoFold/chr13
mkdir -p evoFold/chr14
mkdir -p evoFold/chr15
mkdir -p evoFold/chr16
mkdir -p evoFold/chr17
mkdir -p evoFold/chr18
mkdir -p evoFold/chr19
mkdir -p evoFold/chr2
mkdir -p evoFold/chr20
mkdir -p evoFold/chr21
mkdir -p evoFold/chr22
mkdir -p evoFold/chr3
mkdir -p evoFold/chr4
mkdir -p evoFold/chr5
mkdir -p evoFold/chr6
mkdir -p evoFold/chr7
mkdir -p evoFold/chr8
mkdir -p evoFold/chr9
mkdir -p evoFold/chrM
mkdir -p evoFold/chrX
mkdir -p evoFold/chrY

# get latest verion of the .jar file of VARNA

wget --timestamping http://varna.lri.fr/bin/VARNAv3-7.jar

# Create Java command line files

echo 'doEvoFold hg19 do$1 $1' >do1Chrom
chmod +x do1Chrom

do1Chrom chr1
do1Chrom chr10
do1Chrom chr11
do1Chrom chr12
do1Chrom chr13
do1Chrom chr14
do1Chrom chr15
do1Chrom chr16
do1Chrom chr17
do1Chrom chr18
do1Chrom chr19
do1Chrom chr2
do1Chrom chr20
do1Chrom chr21
do1Chrom chr22
do1Chrom chr3
do1Chrom chr4
do1Chrom chr5
do1Chrom chr6
do1Chrom chr7
do1Chrom chr8
do1Chrom chr9
do1Chrom chrM
do1Chrom chrX
do1Chrom chrY

# run the dochrXX command files in small batches with '&' to exploit multiple CPU
# wait an hour for each batch to finish so that we don't suck in too much computational resources.

dochr1 &
dochr2 &
dochr3 &
dochr4 &
dochr5 &

sleep 3600

dochr6 &
dochr7 &
dochr8 &
dochr9 &
dochr10 &

sleep 3600

dochr11 &
dochr12 &
dochr13 &
dochr14 &
dochr15 &

sleep 3600

dochr16 &
dochr17 &
dochr18 &
dochr19 &
dochr20 &

sleep 3600

dochr21 &
dochr22 &
dochrX &
dochrY &
dochrM &

# check the resulting .png files

# create a simple script file, check1, with the following 3 lines:

echo $1
hgsql hg19 -N -e "select count(*) from evofold where chrom='${1}'"
ls evoFold/$1/*.png|wc

chmod +x check1

# create another script file, checkAll, with the following lines:

check1 chr1
check1 chr10
check1 chr11
check1 chr12
check1 chr13
check1 chr14
check1 chr15
check1 chr16
check1 chr17
check1 chr18
check1 chr19
check1 chr2
check1 chr20
check1 chr21
check1 chr22
check1 chr3
check1 chr4
check1 chr5
check1 chr6
check1 chr7
check1 chr8
check1 chr9
check1 chrM
check1 chrX
check1 chrY

chmod +x checkAll
checkAll >j.check

# examing the resuls in j.check to make sure things are OK.

# create symbolic links

ln -s /hive/data/genomes/hg19/bed/evofold/doEvoFold/evoFold  /gbdb/hg19/evoFold 
ln -s /gbdb/hg19/evoFold /usr/local/apache/htdocs/evoFold/hg19

##########################################################################
# Build targetScanS track - (DONE - 2010-12-13 galt)
#       requested by: George Bell gbell at wi.mit.edu
    ssh hgwdev
    mkdir -p /cluster/data/hg19/bed/targetScanS
    cd /cluster/data/hg19/bed/targetScanS

    wget --timestamping http://www.targetscan.org/vert_50/ucsc/hg19/hg19Cons_ALL_CHRS.BED

    hgLoadBed hg19 targetScanS hg19Cons_ALL_CHRS.BED
    #	Loaded 54199 elements of size 6
    featureBits hg19 targetScanS
    #   354163 bases of 2897316137 (0.012%) in intersection

    # Create/edit/check in targetScans.html and trackDb.ra under
    # kent/src/hg/makeDb/trackDb/human/hg19

##########################################################################
# Neandertal tracks for hg19 (DONE - 2010-12-14 - Hiram)
    # data supplied by Ed Green into /hive/data/outside/homNea/hg19
    # add Neandertal group to hg19 grp
    hgsql hg19 -e \
      "insert into grp values ('neandertal', 'Neandertal Assembly and Analysis', 6.5, 1);"

    mkdir -p /hive/data/genomes/hg19/bed/homNea/seqAlis
    cd /hive/data/genomes/hg19/bed/homNea/seqAlis

for T in Feld1 Mez1 Sid1253 Vi33.16 Vi33.25 Vi33.26
do
    ln -s /hive/data/outside/homNea/hg19/${T}.hg18.bam \
	./SL${T}.hg19.bam
done

    ln -s /hive/data/outside/homNea/hg19/*.bam .
    for F in *.bam
do
    samtools index $F
done

    mkdir -p /gbdb/hg19/neandertal/seqAlis
    ln -s `pwd`/SL*.b* /gbdb/hg19/neandertal/seqAlis/

for T in Feld1 Mez1 Sid1253
do
    hgBbiDbLink hg19 bamSL${T} \
	/gbdb/hg19/neandertal/seqAlis/SL${T}.hg19.bam
done
for T in 16 25 26
do
    hgBbiDbLink hg19 bamSLVi33dot${T} \
	/gbdb/hg19/neandertal/seqAlis/SLVi33.${T}.hg19.bam
done

##########################################################################
# DECIPHER, RGD QTL, RGD RAT QTL (MAYBE DONE, Andy 2010-12-13)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir decipher rgdQtl rgdRatQtl
for tab in decipher rgdQtl rgdRatQtl; do 
  echo "select * from "$tab | hgsql hg18 | tail -n +2 | cut -f2- > ${tab}/hg18.bed
  liftOver ${tab}/hg18.bed /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz ${tab}/hg19.{bed,unmapped}
  hgLoadBed hg19 $tab ${tab}/hg19.bed
done
rm bed.tab
wc -l {decipher,rgdQtl,rgdRatQtl}/hg1{8,9}.bed
#  4227 decipher/hg18.bed
#  4048 decipher/hg19.bed
#   254 rgdQtl/hg18.bed
#   225 rgdQtl/hg19.bed
#  6033 rgdRatQtl/hg18.bed
#  5804 rgdRatQtl/hg19.bed
## This isn't very good.  In each case, the unmapped % is over 2%.
## DECIPHER: 95.8%, RGD QTL: 88.6%, RGD RAT QTL: 96.2%
## update for rgdQtl:
hgsqldump hg18 rgdQtlLink | hgsql hg19 
hgsqldump hg18 rgdRatQtlLink | hgsql hg19

#############################################################################
# FISH CLONES LIFT (DONE, Andy 2010-12-14)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir fishClones
cd fishClones/
echo "select * from fishClones" | hgsql hg18 | tail -n +2 > hg18.bed5p
liftOver -bedPlus=5 hg18.bed5p /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.bed5p hg19.unmapped 
wc -l *.bed5p
#   9788 hg18.bed5p
#   9758 hg19.bed5p
grep -i split hg19.unmapped  | wc -l
# 17
grep -i partially  hg19.unmapped  | wc -l
# 13
cp ~/kent/src/hg/lib/fishClones.sql .
hgLoadBed -tab -sqlTable=fishClones.sql -notItemRgb hg19 fishClones hg19.bed5p

#############################################################################
# CGAP SAGE LIFT (DONE, Galt 2010-12-16)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir cgapSage
cd cgapSage
echo "select * from cgapSage" | hgsql hg18 -N > hg18.bed8p
liftOver -tab -bedPlus=8 -hasBin hg18.bed8p /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.bed8p hg19.unmapped 
wc -l *.bed8p
#  276905 hg18.bed8p
#  276865 hg19.bed8p
grep -i split hg19.unmapped  | wc -l
# 0
grep -i partially  hg19.unmapped  | wc -l
# 3
cp ~/kent/src/hg/lib/cgapSage/cgapSage.sql .
hgLoadBed -tab -hasBin -sqlTable=cgapSage.sql -notItemRgb hg19 cgapSage hg19.bed8p

# no lift needed for the lib table
echo "select * from cgapSageLib" | hgsql hg18 -N > cgapSageLib.tab
cp ~/kent/src/hg/lib/cgapSage/cgapSageLib.sql .
hgLoadSqlTab hg19 cgapSageLib cgapSageLib.sql cgapSageLib.tab

#############################################################################
# LASTZ Zebrafish DanRer7 (DONE - 2010-12-17 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzDanRer7.2010-12-17
    cd /hive/data/genomes/hg19/bed/lastzDanRer7.2010-12-17

    cat << '_EOF_' > DEF
# human vs X. zebrafish
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Zebrafish danRer7
SEQ2_DIR=/scratch/data/danRer7/danRer7.2bit
SEQ2_LEN=/scratch/data/danRer7/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LAP=0
SEQ2_LIMIT=40

BASE=/hive/data/genomes/hg19/bed/lastzDanRer7.2010-12-17
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	Elapsed time: 1698m29s
    cat fb.hg19.chainDanRer7Link.txt 
    #	80849592 bases of 2897316137 (2.790%) in intersection

    #	running the swap
    mkdir /hive/data/genomes/danRer7/bed/blastz.hg19.swap
    cd /hive/data/genomes/danRer7/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzDanRer7.2010-12-17/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-swap > swap.log 2>&1 &
    #	real    42m52.920s
    cat fb.danRer7.chainHg19Link.txt 
    #	86716552 bases of 1409770109 (6.151%) in intersection

##############################################################################
# UMass Med School brain histone ChIP-seq 
# (DONE - 2010-12-17 - Kate)
#
# From Troy Whitfield, submitting for Zhiping Weng, collab with Akbarian
# Published in PNAS, April 2010
# Variables: cell, sample, sex, age
# Tissue:  Prefrontal cortex (PFC)
#
# 11 individuals, age .5 to 69 years
# 13 bigWigs of H3K4me3 enrichment: 11 neuronal, 2 non-neuronal 
# 3 peaks files (bed5FloatScore)
# Neuronal cells selected by FACS sorting, based on NeuN marker (NeuN+)
# Non-neuronal (NeuN-) cells are largely glia, microglia, and endothelium
# 3 peak files

# Note: used publicly available blood cell (lymphocyte)
# ChIP-seq as controls:
# K562, GM12878 (from Bernstein ENCODE group), CD4+ (from Barski, HLB)

    cd /hive/data/genomes/hg19/bed

    mkdir uMassBrainHistone
    cd uMassBrainHistone

    wget http://zlab.umassmed.edu/~whitfiet/hg19/uMassBrainHist.tar.gz
    tar xvfz uMassBrainHist.tar.gz
    cd data

    set t = "uMassBrainHistone"

# Load peak tracks
# peaks in neuron not in  blood
    hgLoadBed hg19 -noNameIx -renameSqlTable \
      -sqlTable=/cluster/bin/sqlCreate/bed5FloatScore.sql \
      ${t}PeaksNeuron 11Neuronal_vs_3Blood_hg19.bed
# Loaded 7947 elements of size 6

# peaks in infants (<1 year), not in seniors (>60)
    hgLoadBed hg19 -noNameIx -renameSqlTable \
      -sqlTable=/cluster/bin/sqlCreate/bed5FloatScore.sql \
      ${t}PeaksInfant 3Young_vs3Old_hg19.bed
# Loaded 1292 elements of size 6

# peaks specific to individuals
    hgLoadBed hg19 -noNameIx -renameSqlTable \
      -sqlTable=/cluster/bin/sqlCreate/bed5FloatScore.sql \
      ${t}PeaksSample SampleSpecific_hg19.bed
# Loaded 3214 elements of size 6

# Load signal tracks
# Get metadata from .ddf file

    cd ..
    cat uMassMedBrainHist.ddf

#files   view    sample  cell    sex     age
#data/s6.bw      Signal  6       neuron  male    4.7
#data/s6n.bw     Signal  6       non-neuron      male    4.7
#data/s2.bw      Signal  2       neuron  male    0.58
#data/s3.bw      Signal  3       neuron  female  0.75
#data/s1.bw      Signal  1       neuron  male    0.5
#data/s9.bw      Signal  9       neuron  female  68
#data/s5.bw      Signal  5       neuron  female  2.8
#data/s11.bw     Signal  11      neuron  female  69
#data/s11n.bw    Signal  11      non-neuron      female  69
#data/s7.bw      Signal  7       neuron  male    8.8
#data/s8.bw      Signal  8       neuron  male    14
#data/s10.bw     Signal  10      neuron  female  69
#data/s4.bw      Signal  4       neuron  male    1.3
#data/11Neuronal_vs_3Blood_hg19.bed      Peaks   N/A     N/A     N/A     N/A
#data/3Young_vs3Old_hg19.bed     Peaks   N/A     N/A     N/A     N/
#data/SampleSpecific_hg19.bed    Peaks   N/A     N/A     N/A     N/A


# Generate table names from DDF
# Format:  uMassBrainHistoneSignalS<sample>Neu<P|M><age>yrs<M|F>
# e.g. uMassBrainHistoneSignalS1185Neu4pt7yrsM

    grep Signal uMassMedBrainHist.ddf | \

    cat << 'EOF' > list.pl
while (<>) {
    ($file, $view, $sample, $cell, $sex, $age) = split;
    next unless $view eq 'Signal';
    $cell = ($cell eq 'neuron' ? 'P' : 'M');
    $age =~ s/\./pt/;
    $sex = ($sex eq 'male' ? 'M' : 'F');
    $table = "uMassBrainHistoneSignalS" . $sample . "Neu" . $cell . $age . "yrs" . $sex;
    print $file . "." . $table . "\n";
}
'EOF'

    cat << 'EOF' > load.csh
set gbdb = "/gbdb/hg19/bbi/uMassBrainHistone"
foreach x (`perl list.pl < uMassMedBrainHist.ddf`)
    set f = $x:r
    set t = $x:e
    echo "Loading $f into $t"
    bigWigInfo $f
    ln -s `pwd`/$f $gbdb/$t.bw
    hgBbiDbLink hg19 $t $gbdb/$t.bw
end
'EOF'

    csh load.csh >&! load.out &

#############################################################################
# SwitchDB TSS Track (DONE 2010-12-17 galt)
#
# This liftover is tricky because of the gmStart and gmEnd, which
# are not lifted automatically.  
# The gm coordinates have to be lifted separately.

ssh hgwdev
mkdir /cluster/data/hg19/bed/switchDbTss
cd /cluster/data/hg19/bed/switchDbTss
ln -s /cluster/data/hg17/bed/switchDbTss/hg17.bed hg17.bed
liftOver -tab -bedPlus=6 hg17.bed /gbdb/hg17/liftOver/hg17ToHg19.over.chain.gz hg19.bed unMapped
cat unMapped | grep '^#' | sort | uniq -c
#     61 #Deleted in new

ln -s ~/kent/src/hg/lib/switchDbTss.sql
hgLoadBed -renameSqlTable -sqlTable=switchDbTss.sql hg19 switchDbTssTemp hg19.bed

mysql> select count(*) from switchDbTssTemp;
+----------+
| count(*) |
+----------+
|   132332 | 
+----------+

hgsql -N hg19 -e "select distinct chrom, gmChromStart, gmChromEnd, gmName from switchDbTssTemp" > gmLoc.hg17.bed4
liftOver -tab -bedPlus=4 gmLoc.hg17.bed4 /gbdb/hg17/liftOver/hg17ToHg19.over.chain.gz gmLoc.hg19.bed4 gmUnMapped

cat gmUnMapped | grep '^#' | sort | uniq -c
#      1 #Deleted in new
#     58 #Partially deleted in new
#     57 #Split in new

hgLoadBed hg19 switchDbTssGmLocTemp gmLoc.hg19.bed4


hgsql hg19 < switchDbTss.sql

hgsql hg19 -e "insert into switchDbTss select a.bin, a.chrom, a.chromStart, a.chromEnd, a.name, a.score, a.strand, a.confScore, a.gmName, b.chromStart as gmChromStart, b.chromEnd as gmChromEnd, a.isPseudo from switchDbTssTemp a, switchDbTssGmLocTemp b where a.gmName = b.name"

mysql> select count(*) from switchDbTss;
+----------+
| count(*) |
+----------+
|   131780 | 
+----------+

hgsql hg19 -e "drop table switchDbTssTemp"
hgsql hg19 -e "drop table switchDbTssGmLocTemp"

#############################################################################

#############################################################################
# FOSMID END PAIRS LIFT FROM HG18 (DONE 2010-12-28, Andy)

mkdir /hive/data/genomes/hg19/bed/hg18MassiveLift/fosEndPairs
cd /hive/data/genomes/hg19/bed/hg18MassiveLift/fosEndPairs/
echo "select * from fosEndPairs" | hgsql hg18 | tail -n +2 | cut -f2- > hg18.fosEndPairs.fep.bed
echo "select * from all_fosends" | hgsql hg18 | tail -n +2 | cut -f2- > hg18.all_fosends.psl
# Converting to bed 12 because of the positional info in nonstandard fields.
# the awk script is pretty simple and is in the directory.
awk -f toBed12.awk hg18.fosEndPairs.fep.bed > hg18.fosEndPairs.bed12
liftOver -bedPlus=12 hg18.fosEndPairs.bed12 /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.fosEndPairs.{bed12,unmapped12}
liftOver -pslT hg18.all_fosends.psl /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.{psl,unmapped}
# remove pairs which have one (or both) ends not in the lifted set of all ends
cut -f14 hg19.fosEndPairs.bed12 | tr ',' '\n' | sort > hg19.fosEndPairs.names
cut -f10 hg19.all_fosends.psl | sort > hg19.all_fosends.names
grep -Fxv -f hg19.all_fosends.names hg19.fosEndPairs.names > bad.names
grep -Fv -f bad.names hg19.fosEndPairs.bed12 > hg19.fosEndPairs.good.bed12
wc -l hg19*.bed12
#  384635 hg19.fosEndPairs.bed12
#  384442 hg19.fosEndPairs.good.bed12
## so... 193 bad ones were removed.  These would have caused join errors if they were left in.

## convert back to fosEndPairs bed6+
awk -f toFosEndPairs.awk hg19.fosEndPairs.good.bed12 > hg19.fosEndPairs.fep.bed

cp ~/kent/src/hg/lib/fosEndPairs.sql .
hgLoadBed -sqlTable=fosEndPairs.sql -notItemRgb hg19 fosEndPairs hg19.fosEndPairs.fep.bed 
hgLoadPsl -table=all_fosends hg19 hg19.all_fosends.psl

wc -l *.fep.bed
#  386129 hg18.fosEndPairs.fep.bed
#  384442 hg19.fosEndPairs.fep.bed
## 99.6% lifted

## Now we need the sequences from all_fosends to be loaded into the seq table in hg19

mkdir /gbdb/hg19/fosends
ln -s /gbdb/hg18/fosends/fosEnds.fa /gbdb/hg19/fosEnds.fromHg18.fa 
hgLoadSeq hg19 /gbdb/hg19/fosends/fosEnds.fromHg18.fa

#############################################################################
# DECIPHER LIFT FORM HG18 (DONE 2010-12-27, Andy)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift/
mkdir decipher
cd decipher/
hgsql -e "select * from decipherRaw" hg18 | tail -n +2 > hg18.decipherRaw.txt
cat hg18.decipherRaw.txt | awk 'BEGIN{FS="\t";OFS="\t"}{ chr="chr"$4; $4=$1; $1=chr; $2 = $2 - 1; print;}' | liftOver -bedPlus=4 -tab stdin /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz stdout hg19.decipherRaw.unmapped | sed 's/^chr//' | awk 'BEGIN{FS="\t";OFS="\t"}{ t=$1; $1=$4; $4=t; print;}' > hg19.decipherRaw.txt
cp ~/kent/src/hg/lib/decipherRaw.sql .
hgLoadSqlTab hg19 decipherRaw decipherRaw.sql hg19.decipherRaw.txt
hgsql -e "select * from decipher" hg18 | tail -n +2 | cut -f2- > hg18.decipher.bed
liftOver hg18.decipher.bed /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.decipher.{bed,unmapped}
cut -f1 hg19.decipherRaw.txt > hg19.decipherRaw.names
cut -f4 hg19.decipher.bed > hg19.decipher.names
# how many of the lifted deciphers are not in the lifted decipherRaws?
grep -Fvx -f hg19.decipher.names hg19.decipherRaw.names | wc -l
#0
# none. ok then, we are done.
rm *.names
wc -l *.bed
#  4227 hg18.decipher.bed
#  4048 hg19.decipher.bed
hgLoadBed hg19 decipher hg19.decipher.bed

#############################################################################
# CLONE COVERAGE LIFT FROM HG18 (DONE 2010-12-29, Andy)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift/
mkdir clonePos
cd clonePos/
hgsql --skip-column-names -e "select * from clonePos" hg18 | awk 'BEGIN{FS="\t"; OFS="\t";}{print $4, $5, $6, $1, $2, $3, $7, $8;}' > hg18.clonePos.bed4p
liftOver -tab -bedPlus=4 hg18.clonePos.bed4p /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.clonePos.{bed4p,unmapped}
awk 'BEGIN{FS="\t"; OFS="\t";}{print $4, $5, $6, $1, $2, $3, $7, $8;}' hg19.clonePos.bed4p > hg19.clonePos.txt
hgLoadSqlTab hg19 clonePos clonePos.sql hg19.clonePos.txt
## it loaded but there seems to be a dependency on the "chr*_gl" tables.  sigh..
mkdir gl
cd gl/
echo "show tables like 'chr%_gl'" | hgsql hg18 | tail -n +2 | while read table; do echo "select * from "$table | hgsql hg18 | tail -n +2 | cut -f2- | awk -v chr=${table%_gl} 'BEGIN{OFS="\t"}{print chr, $2, $3, $1, "1000", $4;}'; done > hg18.gl.bed
liftOver hg18.gl.bed /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.gl.{bed,unmapped}
hgLoadBed -noLoad hg19 gl hg19.gl.bed
mv bed.tab hg19.gl.withBin.bed
awk 'BEGIN{OFS="\t";}{fname=$2"_gl.txt"; print $1, $5, $3, $4, $7 >> fname;}' hg19.gl.withBin.bed
cd ../
for tab in `echo show tables like "'chr%_gl'" | hgsql hg19 | tail -n +2`; do 
  echo select frag from $tab | hgsql hg19 | tail -n +2  >> gl.names; 
done
sed 's/_.*//' gl.names | sort | uniq > uniq.gl.names
cut -f4 hg19.clonePos.bed4p > hg19.clonePos.names
diff uniq.gl.names hg19.clonePos.names | grep '<' | sed 's/< //' > bad_gl.names
for f in chr*.txt; do 
   tab=${f%.txt}
   grep -v -f ../bad_gl.names $f > ${tab}.update.txt
   hgLoadSqlTab hg19 ${tab} hg18.chr1_gl.sql ${tab}.update.txt; 
done

#############################################################################
# MGI MOUSE QTL LIFT (DONE 2010-12-30, Andy)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir mgiMouseQtl
cd mgiMouseQtl/
## There are two subtracks to deal with but it's not a big deal.
## Both are bed4.
for tab in jaxQtlAsIs jaxQtlPadded; do
   hgsql hg18 --skip-column-names -e "select chrom,chromStart,chromEnd,name from "$tab > hg18.${tab}.bed
   liftOver hg18.${tab}.bed /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.${tab}.{bed,unmapped}
   hgLoadBed hg19 $tab hg19.${tab}.bed
done
wc -l *.bed
#    398 hg18.jaxQtlAsIs.bed
#   1463 hg18.jaxQtlPadded.bed
#    383 hg19.jaxQtlAsIs.bed
#   1462 hg19.jaxQtlPadded.bed
## 96.2% for jaxQtlAsIs, 99.9% for jaxQtlPadded.

#############################################################################
# H-INV LIFT (DONE 2010-12-30, Andy)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir HInv
cd HInv/
hgsql hg18 --skip-column-names -e "select * from HInvGeneMrna" | cut -f2- > hg18.HInvGeneMrna.psl
liftOver -pslT hg18.HInvGeneMrna.psl /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.HInvGeneMrna.{psl,unmapped}
hgLoadPsl -table=HInvGeneMrna hg19 hg19.HInvGeneMrna.psl
## A couple non-positional tables too:
hgsqldump hg18 knownToHInv | hgsql hg19
hgsqldump hg18 HInv | hgsql hg19

#############################################################################
# SIB ALT-SPLICING LIFT (DONE 2010-12-30, Andy)
# Note: 
# Obsolete. See the section "SIB Transcriptome (DONE 2011-12-02 Chin)" 
# down below and redmine track #5630 for more details

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir sibTxGraph
cd sibTxGraph/
## there is positional data in the 9th column, so I'll convert it to a bed12+
## and back to the native format later.  Again, the awk scripts to do that
## are aptly named, and reside in the hg19 directory with the data. 
hgsql hg18 --skip-column-names -e "select * from sibTxGraph" | cut -f2- | awk -f toBed12Plus.awk > hg18.sibTxGraph.bed12p
liftOver -tab -bedPlus=12 hg18.sibTxGraph.bed12p /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.sibTxGraph.{bed12p,unmapped}
awk -f toSib.awk hg19.sibTxGraph.bed12p > hg19.sibTxGraph.txt
cut -f1-3 hg19.sibTxGraph.bed12p > hg19.sibTxGraph.bed3
hgLoadBed -noLoad hg19 sibTxGraph hg19.sibTxGraph.bed3
cut -f1 bed.tab > hg19.sibTxGraph.bins
paste hg19.sibTxGraph.bins hg19.sibTxGraph.txt > hg19.sibTxGraph.withBin.txt
## Oddly, there's no .sql file for this in the kent source-tree, so I'll 
## make one directly from hg18 that's suitable for hgLoadSqlTab
hgsqldump --no-data --compact hg18 sibTxGraph | sed '/^SET/d;s/ENGINE.*//'  > sibTxGraph.sql
hgLoadSqlTab hg19 sibTxGraph sibTxGraph.sql hg19.sibTxGraph.withBin.txt

wc -l *.bed12p
#    47094 hg18.sibTxGraph.bed12p
#    47008 hg19.sibTxGraph.bed12p
## 99.8% lifted, not bad.

#############################################################################
# ILLUMINA WG-6 LIFT TO HG19 (DONE 2010-12-30, Andy)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir illuminaProbes
cd illuminaProbes/
## Just copy the seq table to hg19
hgsqldump hg18 illuminaProbesSeq | hgsql hg19
## Two tables: a PSL and a BED:
hgsql hg18 --skip-column-names -e "select * from illuminaProbes" | cut -f2- > hg18.illuminaProbes.bed
hgsql hg18 --skip-column-names -e "select * from illuminaProbesAlign" | cut -f2- > hg18.illuminaProbesAlign.psl
liftOver hg18.illuminaProbes.bed /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.illuminaProbes.{bed,unmapped}
liftOver -pslT hg18.illuminaProbesAlign.psl /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.illuminaProbesAlign.{psl,unmapped}
hgLoadBed hg19 illuminaProbes hg19.illuminaProbes.bed
hgLoadPsl -table=illuminaProbesAlign hg19 hg19.illuminaProbesAlign.psl
## Just to check the probes and align tables are essentially the same
cut -f4 hg19.illuminaProbes.bed | sort > hg19.illuminaProbes.names
cut -f10 hg19.illuminaProbesAlign.psl | sort > hg19.illuminaProbesAlign.names
diff *.names
#(no output)
wc -l *.bed *.psl
#   44163 hg18.illuminaProbes.bed
#   44088 hg19.illuminaProbes.bed
#   44163 hg18.illuminaProbesAlign.psl
#   44088 hg19.illuminaProbesAlign.psl
## 99.8% lifted

#############################################################################
# EIO/JCVI NAS LIFT TO HG19 (DONE 2010-12-30, Andy)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir eioJcviNAS
cd eioJcviNAS/
for tab in eioJcviNASNeg eioJcviNASPos; do
   hgsql hg18 --skip-column-names -e "select * from "$tab > hg18.${tab}.bed 
   liftOver hg18.${tab}.bed /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.${tab}.{bed,unmapped}
   hgLoadBed hg19 ${tab} hg19.${tab}.bed
done
wc -l *.bed 
#  338278 hg18.eioJcviNASNeg.bed
#  130535 hg18.eioJcviNASPos.bed
#  338238 hg19.eioJcviNASNeg.bed
#  130504 hg19.eioJcviNASPos.bed
## > 99.9% of items lifted in both tables: pretty good.
## One strange thing about this one is that the hg18 tables don't have a bin
## field.  I doubt it's important to keep it that way.

#############################################################################
# ORegAnno LIFT TO HG19 (DONE 2010-12-31, Andy)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir oreganno
cd oreganno
hgsql hg18 --skip-column-names -e "select * from oreganno" | cut -f2- > hg18.oreganno.bed3p
liftOver -bedPlus=3 -tab hg18.oreganno.bed3p /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.oreganno.{bed3p,unmapped}
cp ~/kent/src/hg/lib/oreganno.sql .
hgLoadBed -tab -sqlTable=oreganno.sql hg19 oreganno hg19.oreganno.bed3p
hgsqldump hg18 oregannoLink | hgsql hg19
hgsqldump hg18 oregannoAttr | hgsql hg19
wc -l *.bed3p
#  23130 hg18.oreganno.bed3p
#  23118 hg19.oreganno.bed3p
## 99.9% lifted.

#############################################################################
# NK NUC LAMINA LIFT TO HG19 (DONE 2010-12-31, Andy)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir laminB1
cd laminB1/
hgsql hg18 --skip-column-names -e "select * from laminB1Lads" | cut -f2- > hg18.laminB1Lads.bed
liftOver hg18.laminB1Lads.bed /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.laminB1Lads.{bed,unmapped}
hgLoadBed hg19 laminB1Lads hg19.laminB1Lads.bed
wc -l *.bed
# 1344 hg18.laminB1Lads.bed
# 1302 hg19.laminB1Lads.bed
## 96.9% lifted... ok, not bad I guess.

ln -s /hive/data/genomes/hg18/bed/nuclearLamina/LaminB1_080513.wig hg18.laminB1.customWigVarStep
awk -f toBedGraph.awk hg18.laminB1.customWigVarStep > hg18.laminB1.bg
liftOver -bedPlus=3 -tab hg18.laminB1.bg /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.laminB1.{bg,unmapped}
awk '{if ($3 - $2 == 60) print;}' hg19.laminB1.bg | sort -k1,1 -k2,2n | awk 'BEGIN{prev=-100;chrom="";FS="\t";OFS="\t";}{ if ((chrom != $1) || ($2 - prev > 60)) {print; chrom = $1; prev = $2;}}'> hg19.laminB1.span60.bg
wigBedToStep hg19.laminB1.span60.bg hg19.laminB1.span60.wigVarStep
ln -s hg19.laminB1.span60.wigVarStep laminB1.wig
wigEncode laminB1.{wig,wiggle,wib}
#Converted laminB1.wig, upper limit 5.68, lower limit -6.60
ln -s `pwd`/laminB1.wib /gbdb/hg19/wib/laminB1.wib
hgLoadWiggle hg19 laminB1 laminB1.wiggle
wc -l hg18.laminB1.bg hg19.laminB1.span60.bg
#  2909178 hg18.laminB1.bg
#  2908692 hg19.laminB1.span60.bg
## In total, 99.98% of the datapoints lifted cleanly.

#############################################################################
# UCSF BRAIN METHYLATION (DONE 2010-12-31, Andy)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir ucsfBrainMethyl
cd ucsfBrainMethyl/

## 10 tables:
##
## ucsfChipSeqH3K4me3BrainCoverage    (bedGraph/bed3+)
## ucsfMreSeqBrainReads               (bed9)
## ucsfMreSeqBrainCpG                 (bedGraph/bed3+)
## ucsfMedipSeqBrainReads             (bed9)
## ucsfMedipSeqBrainCpG               (bedGraph/bed3+)
## ucsfMedipSeqBrainCoverage          (bedGraph/bed3+)
## ucsfRnaSeqBrainAllReads            (bed9)
## ucsfRnaSeqBrainAllCoverage         (bedGraph/bed3+)
## ucsfRnaSeqBrainSmartReads          (bed9)
## ucsfRnaSeqBrainSmartCoverage       (bedGraph/bed3+)

## Do the bed9s first:

for tab in ucsfMreSeqBrainReads ucsfMedipSeqBrainReads ucsfRnaSeqBrainAllReads ucsfRnaSeqBrainSmartReads; do
   hgsql hg18 --skip-column-names -e "select * from "$tab | cut -f2- > hg18.${tab}.bed
   liftOver hg18.${tab}.bed /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.${tab}.{bed,unmapped}
   hgLoadBed hg19 $tab hg19.${tab}.bed
   wc -l *.${tab}.bed
done

#   10110644 hg18.ucsfMreSeqBrainReads.bed
#   10109979 hg19.ucsfMreSeqBrainReads.bed
## 99.99% lifted
#   44130143 hg18.ucsfMedipSeqBrainReads.bed
#   44120612 hg19.ucsfMedipSeqBrainReads.bed
## 99.98% lifted
#   63033692 hg18.ucsfRnaSeqBrainAllReads.bed
#   63031432 hg19.ucsfRnaSeqBrainAllReads.bed
#   26767318 hg18.ucsfRnaSeqBrainSmartReads.bed
#   26766288 hg19.ucsfRnaSeqBrainSmartReads.bed
## getting old now, we get it... it lifts.
    
for tab in ucsfChipSeqH3K4me3BrainCoverage ucsfMreSeqBrainCpG ucsfMedipSeqBrainCpG ucsfMedipSeqBrainCoverage ucsfRnaSeqBrainAllCoverage ucsfRnaSeqBrainSmartCoverage; do
   hgsql hg18 --skip-column-names -e "select * from "$tab | cut -f2- > hg18.${tab}.bg
   liftOver -bedPlus=3 hg18.${tab}.bg /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.${tab}.{bg,unmapped}
   hgLoadBed -bedGraph=4 hg19 $tab hg19.${tab}.bg
   wc -l *.${tab}.bg
done

#  2598517 hg18.ucsfChipSeqH3K4me3BrainCoverage.bg
#  2598085 hg19.ucsfChipSeqH3K4me3BrainCoverage.bg
#  1165599 hg18.ucsfMreSeqBrainCpG.bg
#  1165521 hg19.ucsfMreSeqBrainCpG.bg
# 20862283 hg18.ucsfMedipSeqBrainCpG.bg
# 20859033 hg19.ucsfMedipSeqBrainCpG.bg
# 80960101 hg18.ucsfMedipSeqBrainCoverage.bg
# 80943454 hg19.ucsfMedipSeqBrainCoverage.bg
# 17019268 hg18.ucsfRnaSeqBrainAllCoverage.bg
# 17017461 hg19.ucsfRnaSeqBrainAllCoverage.bg
#  6141663 hg18.ucsfRnaSeqBrainSmartCoverage.bg
#  6140890 hg19.ucsfRnaSeqBrainSmartCoverage.bg
## again in each case, almost all the data in the table lifts.

for f in *; do gzip $f; echo $f zipped; done

## One more thing: remove overlapping items in lifted bedGraphs:

for f in hg19*.bg.gz; do 
    pre=${f%.bg.gz};
    tab=${pre#hg19.};
    echo $tab;
    gunzip -c $f | sort -k1,1 -k2,2n | bedGraphLegalize -report=${pre}.bad.txt stdin stdout | gzip -c > ${pre}.legal.bg.gz
    hgLoadBed -bedGraph=4 hg19 $tab ${pre}.legal.bg.gz
done

#############################################################################
# SNP ARRAYS LIFT TO HG19 (DONE 2010-12-31, Andy)

cd /hive/data/genomes/hg19/bed/hg18MassiveLift
mkdir snpArray
cd snpArray/

## 12 arrays:
## 
## snpArrayAffy6                   (bed6+, 8 fields)
## snpArrayAffy6SV                 (bed6)
## snpArrayAffy5                   (bed6+, 8 fields)
## snpArrayAffy250Nsp              (bed6+, 8 fields)
## snpArrayAffy250Sty              (bed6+, 8 fields)
## snpArrayIllumina650             (bed6+, 7 fields)
## snpArrayIllumina550             (bed6+, 7 fields)
## snpArrayIllumina300             (bed6+, 7 fields)
## snpArrayIllumina1M              (bed6+, 7 fields)
## snpArrayIlluminaHumanCytoSNP_12 (bed6+, 7 fields)
## snpArrayIlluminaHuman660W_Quad  (bed6+, 7 fields)
## snpArrayIlluminaHumanOmni1_Quad (bed6+, 7 fields)

## Get the bed6 one out of the way first:

hgsql hg18 --skip-column-names -e "select * from snpArrayAffy6SV" | cut -f2- > hg18.snpArrayAffy6SV.bed
liftOver hg18.snpArrayAffy6SV.bed /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.snpArrayAffy6SV.{bed,unmapped}
hgLoadBed hg19 snpArrayAffy6SV hg19.snpArrayAffy6SV.bed
wc -l *.bed
#  945805 hg18.snpArrayAffy6SV.bed
#  945615 hg19.snpArrayAffy6SV.bed

## The rest each may or may not have their own module in lib
## For simplicity sake, I'll just dump the CREATEs straight from
## hg18 into their own .sql file.

for table in snpArrayAffy6 snpArrayAffy5 snpArrayAffy250Nsp snpArrayAffy250Sty snpArrayIllumina650 snpArrayIllumina550 snpArrayIllumina300 snpArrayIllumina1M snpArrayIlluminaHumanCytoSNP_12 snpArrayIlluminaHuman660W_Quad snpArrayIlluminaHumanOmni1_Quad; do
   hgsql hg18 --skip-column-names -e "select * from "$table | cut -f2- > hg18.${table}.bed6p
   hgsqldump --no-data --compact hg18 $table | sed '/^SET/d;s/ENGINE.*//' > hg18.${table}.sql
   liftOver -bedPlus=6 hg18.${table}.bed6p /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.${table}.{bed6p,unmapped}
   hgLoadBed -sqlTable=hg18.${table}.sql -renameSqlTable hg19 $table hg19.${table}.bed6p
done
for table in snpArrayAffy6 snpArrayAffy5 snpArrayAffy250Nsp snpArrayAffy250Sty snpArrayIllumina650 snpArrayIllumina550 snpArrayIllumina300 snpArrayIllumina1M snpArrayIlluminaHumanCytoSNP_12 snpArrayIlluminaHuman660W_Quad snpArrayIlluminaHumanOmni1_Quad; do
   hg18=`wc -l hg18.${table}.bed6p | awk '{print $1}'`
   hg19=`wc -l hg19.${table}.bed6p | awk '{print $1}'`
   perc=`echo ${hg19}"/"${hg18}" * 100" | R --vanilla | grep "\[1\]" | awk '{print $2}'`
   printf "%s: %d/%d items lifted (%.3f%%)\n" $table $hg19 $hg18 $perc
done
# snpArrayAffy6: 909297/909508 items lifted (99.977%)
# snpArrayAffy5: 440638/440734 items lifted (99.978%)
# snpArrayAffy250Nsp: 257159/257213 items lifted (99.979%)
# snpArrayAffy250Sty: 233887/233941 items lifted (99.977%)
# snpArrayIllumina650: 660388/660557 items lifted (99.974%)
# snpArrayIllumina550: 560972/561122 items lifted (99.973%)
# snpArrayIllumina300: 318046/318117 items lifted (99.978%)
# snpArrayIllumina1M: 1217520/1219961 items lifted (99.800%)
# snpArrayIlluminaHumanCytoSNP_12: 302127/302402 items lifted (99.909%)
# snpArrayIlluminaHuman660W_Quad: 664655/665901 items lifted (99.813%)
# snpArrayIlluminaHumanOmni1_Quad: 1169872/1175447 items lifted (99.526%)

## Now there's a few "Raw" tables to lift. Convert them to bed3+ first:

for tab in `echo show tables like "'snpArray%Raw'" | hgsql hg18 | tail -n +2`; do 
  hgsql hg18 --skip-column-names -e "select * from "$table | awk 'BEGIN{FS="\t";OFS="\t"}{print "chr"$6, $7 - 1, $7, $1, $2, $3, $4, $5}' > hg18.${table}.bed3p
  liftOver -bedPlus=3 -tab hg18.${table}.bed3p /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.${table}.{bed3p,unmapped}
  awk 'BEGIN{FS="\t";OFS="\t"}{print $4, $5, $6, $7, $8, substr($1, 4), $3;}' hg19.${table}.bed3p > hg19.${table}.txt
  hgsqldump --no-data --compact hg18 $table | sed '/^SET/d;s/ENGINE.*//' > hg18.${table}.sql
  hgLoadSqlTab hg19 $table hg18.${table}.sql hg19.${table}.txt
done

############################################################################
# HAPMAP SNPS AND HAPMAP LD PHASED LIFTS FROM HG18 (Andy)

mkdir /hive/data/genomes/hg19/bed/hg18MassiveLift/hapmapSnps
cd /hive/data/genomes/hg19/bed/hg18MassiveLift/hapmapSnps

## All the tables in the trackDb entry seem to be bed 6 +
for table in `grep -B1 "parent hapmapSnps" ~/kent/src/hg/makeDb/trackDb/human/trackDb.ra | grep track | sed 's/.*track\ //'`; do
   echo $table >> tables.txt;
done
for table in `cat tables.txt`; do 
   hgsql hg18 --skip-column-names -e "select * from "$table | cut -f2- > hg18.${table}.bed6p;
   liftOver -bedPlus=6 -tab hg18.${table}.bed6p /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.${table}.{bed6p,unmapped};
   hgsqldump --no-data --compact hg18 $table | sed '/^SET/d;s/ENGINE.*//' > ${table}.sql
   hgLoadBed -sqlTable=${table}.sql -tab hg19 $table hg19.${table}.bed6p;
   wc -l hg1{8,9}.${table}.bed6p >> lifts.txt
done
## Also need hapmapLd% and hapmapAllelesSummary
for table in `hgsql hg18 --skip-column-names -e "show tables like 'hapmapLd%'"` hapmapAllelesSummary; do
   hgsql hg18 --skip-column-names -e "select * from "$table | cut -f2- > hg18.${table}.bed6p;
   liftOver -bedPlus=6 -tab hg18.${table}.bed6p /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.${table}.{bed6p,unmapped};
   hgsqldump --no-data --compact hg18 $table | sed '/^SET/d;s/ENGINE.*//' > ${table}.sql
   hgLoadBed -sqlTable=${table}.sql -tab hg19 $table hg19.${table}.bed6p;
   wc -l hg1{8,9}.${table}.bed6p >> lifts.txt
done
## Also need hapmapPhaseIIISummary
hgsql hg18 --skip-column-names -e "select * from hapmapPhaseIIISummary" | cut -f2- > hg18.hapmapPhaseIIISummary.bed5p
liftOver -bedPlus=5 hg18.hapmapPhaseIIISummary.bed5p /gbdb/hg18/liftOver/hg18ToHg19.over.chain.gz hg19.hapmapPhaseIIISummary.{bed5p,unmapped}
hgsqldump --no-data --compact hg18 hapmapPhaseIIISummary | sed '/^SET/d;s/ENGINE.*//' > hapmapPhaseIIISummary.sql
hgLoadBed -sqlTable=hapmapPhaseIIISummary.sql hg19 hapmapPhaseIIISummary hg19.hapmapPhaseIIISummary.bed5p


############################################################################
# INDEL-BASED CONSERVATION TRACK liftOver to hg19 (DONE - 2010-12-21 - Chin )
# Data from the Gerton Lunter (gerton.lunter at anat.ox.ac.uk), MRC
# Functional Genetics Unit, University of Oxford, United Kingdom.
# Data is from the paper:
# Lunter G, Ponting CP and Hein J Genome-wide identification of human
# functional DNA using a neutral indel model. PLoS Comput Biol. 2006
# Jan;2(1):e5.
    mkdir -p /hive/data/genomes/hg19/bed/consIndels/data
    cd /hive/data/genomes/hg19/bed/consIndels
    cp /hive/data/genomes/hg18/bed/consIndels/README.indels .
    cp /hive/data/genomes/hg18/bed/consIndels/igs-hg18mm8cf2.zip .
    # 38 Mb zip file in GFF format. This contains data for hg18
    # comparing it to mm8 and cf2 (canFam2).
    unzip igs-hg18mm8cf2.zip
    mv *.gff ./data/
    cd /hive/data/genomes/hg19/bed/consIndels/data    

    for f in *.gff
    do
       echo processing $f
       grep -v "track" $f >> ../allNoHeader.tmp
    done

    cd /hive/data/genomes/hg19/bed/consIndels/	
    cat allNoHeader.tmp | \
    awk '{print $1,$4,$5,$6,$9,$10,$11}' > consIndels.bed7p       

    liftOver -bedPlus=3 consIndels.bed7p \
      /hive/data/genomes/hg18/bed/liftOver/hg18ToHg19.over.chain.gz \
      consIndelsHg19Mm8CanFam2.bed7p unMapped

    wc -l *.bed7p
    #  2603017 consIndels.bed7p
    #  2602701 consIndelsHg19Mm8CanFam2.bed7p
    grep -i split unMapped | wc -l
    # 41
    grep -i partially unMapped | wc -l
    # 66
    rm allNoHeader.tmp
    rm consIndels.bed7p
   
    # strip off the end of the name e.g. IGS0001.1:p=.74; FDR 0.27
    # so that the name displayed is short - IGS0001.1. The score field
    # is used to determine colouring and this is calculated from FDR
    cd /cluster/data/hg19/bed/consIndels
    cat  consIndelsHg19Mm8CanFam2.bed7p | 
      awk '{print $1,$2, $3, $5, $4}' | 
      sed -e 's/:p[=<]\.[0-9][0-9]*;//' \
       > consIndelsHg19Mm8CanFam2.bed

    # load data
    cd /hive/data/genomes/hg19/bed/consIndels
    hgLoadBed hg19 consIndelsHgMmCanFam consIndelsHg19Mm8CanFam2.bed
    # Reading consIndelsHg19Mm8CanFam2.bed
    # Loaded 2602701 elements of size 5
    # Sorted
    # Creating table definition for consIndelsHgMmCanFam
    # Saving bed.tab
    # Loading hg19

    # Get the IDs, posterior probabilities (p) for the segment being
    # neutral,
    # and the FDR from the original GFFs for a separate table. Some
    # items
    # have p<.001. Can not do Table Browser queries restricting
    # p to <, =, or > a specified value unless all values are floats.
    # Contacted the data contributor, Gerton Lunter, and he said it
    # would be
    # ok to change all p<.001 to p=0.0005
    cd /hive/data/genomes/hg19/bed/consIndels/
    cat consIndelsHg19Mm8CanFam2.bed7p \
      | awk 'BEGIN {FS="\t"} {print $5, $6, $7}'  \
      | sed -e 's/:/\t/' \
      | sed -e 's/p=\./0\./' | sed -e 's/p<\.001/0\.0005/' \
      | sed -e 's/;\sFDR/\t/' > consIndelsHg19Mm8CanFam2Conf.txt

    # there are no GFF files for the haplotype chroms
    # Reuse $HOME/kent/src/hg/lib/itemConf.* for the table of identifier,
    # posterior probability and false discovery rate (FDR).
    
    cd /hive/data/genomes/hg19/bed/consIndels
    hgLoadSqlTab hg19 consIndelsHgMmCanFamConf \
         $HOME/kent/src/hg/lib/itemConf.sql \
         consIndelsHg19Mm8CanFam2Conf.txt

    # check that all itesm are in this table.
    hgsql -N -e 'select distinct(name) from consIndelsHgMmCanFam;' hg19 \
         | sort > consIndels.names.sort
    hgsql -N -e 'select distinct(id) from consIndelsHgMmCanFamConf;' hg19 \
         | sort > consIndels.idsfromConf.sort
    wc -l *.sort
    # 2602701 consIndels.idsfromConf.sort
    # 2602701 consIndels.names.sort

    comm -12 consIndels.names.sort consIndels.idsfromConf.sort | wc -l
    # 2602701
    # so all element IDs are in both tables.
    # cleanup
    rm ./data/*.bak *.sort

    # add trackDb/human/hg19/trackDb.ra entry and add description that
    # was written by the data contributor. Add code to hgc.c to display
    # the posterior probability and the FDR on the details page for
    # track elements. Gerton Lunter provided a description for the data
    # on 2007-09-12.
    # Add hg19 to the "identifier consIndelsId" in all.joiner.


############################################################################
# POLYA_DB TRACK (DONE 2011-01-04 - Chin)
#

    # Data files and program:
    # "Bin Tian" <btian@umdnj.edu> provided the following two 
    # data files at /hive/data/outside/polyA:
    #   hg18.polyadb.bed  hg19.polyadb.bed
    # Andy found the SVM program he used before, and in here:
    #  /hive/data/genomes/hg18/bed/polyaDB/polya_svm_2.2.tar.gz
    # Copy it to /hive/data/outside/polyA. Unzip the program to 
    # polya_svm_2.2

    mkdir /hive/data/genomes/hg19/bed/polyaDB
    cd /hive/data/genomes/hg19/bed/polyaDB
    cp /hive/data/outside/polyA/hg19.polyadb.bed .
    hgLoadBed hg19 polyaDb hg19.polyadb.bed
    # add trackDb entry in human/hg19
    # polyA.html is at top human level 

    # since hg19.polyadb.bed provided is lifted over from hg18.polyadb.bed,
    # it it safe to lift polyaPredict table from hg18 to hg19 without re-run
    # the svm.
    hgsql -N -e "select * from polyaPredict;" hg18 | \
      cut -f2-9 > hg18.polyaPredict.bed
    wc -l hg18.polyaPredict.bed
    # 52182 hg18.polyaPredict.bed

    liftOver -bedPlus=8 hg18.polyaPredict.bed \
      /hive/data/genomes/hg18/bed/liftOver/hg18ToHg19.over.chain.gz \
      hg19.polyaPredict.bed unMapped.polyaPrdict
    wc -l hg19.polyaPredict.bed
    # 52169 hg19.polyaPredict.bed
    hgLoadBed hg19 polyaPredict hg19.polyaPredict.bed
    # Reading hg19.polyaPredict.bed
    # Loaded 52169 elements of size 8
    # Sorted
    # Creating table definition for polyaPredict
    # Saving bed.tab
    # Loading hg19


#############################################################################
# FILTER SNP132 (DONE 3/8/11 angie)
# 4/8/11: changing table names to be consistent with shortLabel:
#         snp132Patient -> snp132Flagged
#         snp132NonUnique -> snp132Mult
# 3/8/11: redone after snp132 with tweaked weights in PARs, see above
# Previously done 1/24/11 after snp132
    # Redmine: Track #1684 (SNPs 132 (dbSNP))
    # Make several tracks that are filtered subsets of snp132:
    # First, filter out the multiply-aligned and/or weight >1 SNPs [any other exceptions?]
    cd /hive/data/outside/dbSNP/132/human
    zcat snp132.parWeightTweak.bed.gz \
    | perl -we \
      'open($mult, "| gzip -c > snp132Mult.bed.gz") || die; \
       open($common,    "| gzip -c > snp132Common.bed.gz") || die; \
       open($flagged,   "| gzip -c > snp132Flagged.bed.gz") || die; \
       open($misc,      "| gzip -c > snp132Misc.bed.gz") || die; \
       while (<>) { \
         @w = split("\t"); \
         if ($w[16] > 1 || $w[17] =~ /MultipleAlignments/) { \
           print $mult $_; \
         } else { \
           my ($alleleFreqCount, $nStr, $freqStr) = ($w[20], $w[22], $w[23]); \
           my @alNs = split(",", $nStr);      die unless scalar(@alNs) == $alleleFreqCount; \
           my @freqs = split(",", $freqStr);  die unless scalar(@freqs) == $alleleFreqCount; \
           my ($total2N, $maxAlleleFreq) = (0, 0); \
           for (my $i = 0;  $i < $alleleFreqCount;  $i++) { \
             $total2N += $alNs[$i]; \
             $maxAlleleFreq = $freqs[$i] if ($freqs[$i] > $maxAlleleFreq); \
           } \
           if ($alleleFreqCount >= 2 && $total2N >= 4 && $maxAlleleFreq <= 0.99) { \
             print $common $_; \
           } elsif($w[24] =~ /clinically-assoc/)  { \
             print $flagged $_; \
           } else { \
             print $misc $_; \
           } \
         } \
       } \
       close($mult);  close($common); close($flagged);  close($misc);'
    zcat snp132Mult.bed.gz | wc -l
#3568988
    zcat snp132Common.bed.gz | wc -l
#14024295
    zcat snp132Flagged.bed.gz | wc -l
#18084
    zcat snp132Misc.bed.gz | wc -l
#15414754

    # Load tables
    foreach subset (Mult Common Flagged Misc)
      hgLoadBed -tab -onServer -tmpDir=/data/tmp -allowStartEqualEnd -renameSqlTable \
        hg19 snp132$subset -sqlTable=snp132.sql snp132$subset.bed.gz
    end


#############################################################################
# BUILD DECIPHER WITH NEW HG19 RELEASE (Done Fan, 2/8/11)

# The decipher track is built by an automated process.  The following two scripts:

#   kent/src/utils/decipher/checkDecipher.sh
#   kent/src/utils/decipher/buildDecipher 

#are used to automatically detect update on DECIPHER ftp sites and then
#download and build the decipher track.

# checkDecipher.sh is invoked by a cron job, it will call buildDecipher to build 
# the decipher track after the new data are downloaded.

#############################################################################
# NCBI Incident database (DONE - 2011-02-10 - Hiram)
    # this procedure is run as a cron job in Hiram's account:

    #	43 09 * * * /hive/data/outside/ncbi/incidentDb/runUpdate.sh makeItSo

    # using the two scrips there: runUpdate.sh and update.sh
    # which are checked into the source tree as files:
    #	src/hg/utils/automation/ncbiIncidentUpdate.sh
    #	src/hg/utils/automation/ncbiRunIncidentUpdate.sh

    # they fetch the XML files from NCBI, convert them to SQL text
    # files, construct a bigBed file, and pushes it to genomewiki if
    # it is an update from previous

    # the table in the dataBase is: ncbiIncidentDb
    # which is the URL to the bb file, a single row:
    # http://genomewiki.ucsc.edu/images/8/85/Hg19.ncbiIncidentDb.bb

#############################################################################
# UNIGENE/SAGE TRACK (RE-BUILT - 2011-02-22 Fan)

# Create the uniGene alignments

    # Download of the latest UniGene version is now automated by a
    # cron job -- see /cluster/home/angie/crontab ,
    # /cluster/home/angie/unigeneVers/unigene.csh .

    ssh hgwdev
    mkdir -p /hive/data/genomes/hg19/bed/uniGene/022211
    cd /hive/data/genomes/hg19/bed/uniGene/022211

    set Version = 229

    zcat /hive/data/outside/uniGene/uniGene.$Version/Hs.seq.uniq.gz|\
    sed -e "s#>.*/ug=#>#; s# /len.*##;" > Hs.seq.uniq.simpleHeader.fa

    ssh swarm
    set Version = 229
    mkdir -p /hive/data/genomes/hg19/bed/uniGene/022211/run.blat
    cd /hive/data/genomes/hg19/bed/uniGene/022211/run.blat

    ls -1 /hive/data/genomes/hg19/nib/*.nib > genome.lst
    ls -1S \
    /hive/data/genomes/hg19/bed/uniGene/022211/Hs.seq.uniq.simpleHeader.fa \
      > uniGene.lst
    
    cat << '_EOF_' > template.sub
#LOOP
/cluster/bin/x86_64/blat -repeats=lower -minIdentity=95 ooc=/hive/data/genomes/hg19/11.ooc $(path1) $(path2)  {check out line+ psl/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'

    gensub2 genome.lst uniGene.lst template.sub para.spec
    para create para.spec
    mkdir psl
    para try
    para check
    para push

#Completed: 93 of 93 jobs
#CPU time in finished jobs:      67404s    1123.41m    18.72h    0.78d  0.002 y
#IO & Wait Time:                  5838s      97.29m     1.62h    0.07d  0.000 y
#Average job time:                 788s      13.13m     0.22h    0.01d
#Longest finished job:            5228s      87.13m     1.45h    0.06d
#Submission to last job:          5320s      88.67m     1.48h    0.06d
#Estimated complete:                 0s       0.00m     0.00h    0.00d

    pslSort dirs raw.psl tmp psl >& pslSort.log
    cat raw.psl|\
    pslReps -minCover=0.2 -sizeMatters -minAli=0.965 -nearTop=0.002 \
      stdin hg19.uniGene.pslReps.psl /dev/null
    
    gzip raw.psl

    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/uniGene/022211/run.blat

    hgLoadPsl -table=uniGene_3 hg19 hg19.uniGene.pslReps.psl

    mkdir -p /gbdb/hg19/uniGene
    cd /gbdb/hg19/uniGene

    rm Hs.seq.uniq.simpleHeader.fa
    ln -s \
    /hive/data/genomes/hg19/bed/uniGene/022211/Hs.seq.uniq.simpleHeader.fa \
    Hs.seq.uniq.simpleHeader.fa

# load the sequence

    hgLoadSeq -replace hg19 /gbdb/hg19/uniGene/Hs.seq.uniq.simpleHeader.fa

##############################################################################
# Chimp Lastz run (DONE - 2011-02-23 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzPanTro3.2011-02-22
    cd /hive/data/genomes/hg19/bed/lastzPanTro3.2011-02-22
    cat << '_EOF_' > DEF
# human vs chimp
BLASTZ=lastz
# maximum M allowed with lastz is only 254
BLASTZ_M=254
BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q
BLASTZ_O=600
BLASTZ_E=150
BLASTZ_K=4500
BLASTZ_Y=15000
BLASTZ_T=2

# TARGET: Human Hg19
SEQ1_DIR=/scratch/data/hg19/hg19.2bit
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_IN_CONTIGS=0

# QUERY: Chimp PanTro3
SEQ2_DIR=/scratch/data/panTro3/panTro3.2bit
SEQ2_LEN=/scratch/data/panTro3/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0
SEQ2_IN_CONTIGS=0

BASE=/hive/data/genomes/hg19/bed/lastzPanTro3.2011-02-22
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    screen # use screen to manage this long-running job
    time nice -n +19 doBlastzChainNet.pl `pwd`/DEF -verbose=2 \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-syntenicNet > do.log 2>&1 &
    # problems with memk, after recovery, continue chainMerge:
    time nice -n +19 doBlastzChainNet.pl `pwd`/DEF -verbose=2 \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-continue=chainMerge -syntenicNet > chainMerge.log 2>&1 &
    #	real    103m34.088s

    cat fb.hg19.chainPanTro3Link.txt 
    #	2760939621 bases of 2897316137 (95.293%) in intersection

    # filter with doRecipBest.pl 
    time doRecipBest.pl -workhorse=hgwdev -buildDir=`pwd` \
        hg19 panTro3 > rbest.log 2>&1
    # real    50m49.740s


#	running the swap
    mkdir /hive/data/genomes/panTro3/bed/blastz.hg19.swap
    cd /hive/data/genomes/panTro3/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	-swap /hive/data/genomes/hg19/bed/lastzPanTro3.2011-02-22/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-syntenicNet > swap.log 2>&1 &
#	real    86m49.706s
    cat fb.panTro3.chainHg19Link.txt 
    #	2772816267 bases of 2900529764 (95.597%) in intersection

############################################################################
# MAKE tfbsConsSites and tfbsConsFactors for TFBS conserved track (DONE weirauch braney 03/07/11)
# Questions?  braney at soe.ucsc.edu

ssh hgwdev
mkdir /cluster/data/hg19/bed/tfbsCons
cd /cluster/data/hg19/bed/tfbsCons

# Define all parameters in 'PARAMS.txt'
# Define all chromosomes in 'CHROMS.txt'
# Get tfbsConsUtils.tar.gz from Matt Weirauch with Perl scripts  weirauch@soe.ucsc.edu
set tarfile=/cluster/data/hg19/bed/tfbsCons/tfbsConsUtils.tar.gz
tar zxf $tarfile

nice ./getRefseqStats.pl &
nice ./getBatchQueries.pl &

ssh swarm
mkdir /cluster/bluearc/braney/tfloc
mkdir /hive/users/weirauch/tfloc_hg18
# Copy ./tmp/ctfbs_batch_list.txt to this dir
# Copy ./scripts/doit to this dir
para create ctfbs_batch_list.txt
para try
para push

# When the run is done (within a day or so), the results will be in individual dirs, one for each chromosome.

nice ./getBedFile.pl &

hgLoadBed -noSort hg19  tfbsConsSites -sqlTable=$HOME/kent/src/hg/lib/tfbsConsSites.sql tfbsConsSites.bed -tab
hgLoadSqlTab hg19 tfbsConsFactors $HOME/kent/src/hg/lib/tfbsConsFactors.sql tfbsConsFactors.bed
#########################################################################
# BUILD THE TRACK OF IKMC MAPPED TO HUMAN GENOME. (DONE, Fan, 3/15/11)

    ssh hgwdev
    mkdir -p /hive/data/genomes/hg19/bed/ikmc/110314
    cd /hive/data/genomes/hg19/bed/ikmc/110314

# recieve  20110301_human.gff.gz from Carol Bult [Carol.Bult@jax.org] and place it under this subdirectory.
    gzip -d 20110301_human.gff.gz

# build hgIkmc table from raw data file ucschuman.gff (substitue some
# troublesome chroms in raw data file and remove some records mapped to 'chrUn')

    cat 20110301_human.gff |sort -u \
    |sed -e 's/chr9|NT_113911.1/chr9/' \
    |grep -v 'chrUn' \
    | perl -we \
      'while (<>) { \
         s/\r?\n$//; \
         ($chr, undef, $ctr, $s, $e, undef, undef, undef, $id, $col, $n) = split("\t"); \
         if ($s eq "") { warn "$_\n";  s/^.*//; next; } # Some lines have no coords. \
         $col = ($col eq "Yellow") ? "255,215,0" : \
                ($col eq "Green")  ? "0,240,0" : \
                ($col eq "Blue")   ? "0,0,200" : "0,0,0"; \
         $s--; \
         $id =~ s/^MGI:\d+; (\w+); .*/$1/ || die "Cant parse id \"$id\""; \
         my $geneId = join("|", $chr, $ctr, "${n}_$id"); \
         push @{$geneBlks{$geneId}}, [$s, $e, $col] unless $e <= 0; \
      } \
      warn "Got " . scalar(keys %geneBlks) . " genes.\n"; \
      foreach my $geneId (keys %geneBlks) { \
         my @blks = @{$geneBlks{$geneId}}; \
         my ($chrom, $center, $name) = split(/\|/, $geneId); \
         my $blkCount = @blks; \
         @blks = sort {$a->[0] <=> $b->[0]} @blks; \
         my $chromStart = $blks[0]->[0]; \
         my $chromEnd = $blks[$blkCount-1]->[1]; \
         my $color = $blks[0]->[2]; \
         my $blkStarts = ""; \
         my $blkSizes = ""; \
         foreach my $blk (@blks) { \
           my ($start, $end, $col) = @{$blk}; \
           $blkStarts .= ($start - $chromStart) . ","; \
           $blkSizes  .= ($end - $start) . ","; \
           if ($col ne $color) { die "Blocks of $geneId of colors $color and $col"; } \
         } \
        print join("\t", $chrom, $chromStart, $chromEnd, $name, 0, ".", $chromStart, \
                   $chromStart, $color, $blkCount, $blkSizes, $blkStarts) . "\n"; \
      }' \
    | sort -k 1,1 -k 2n,2n > hgIkmc.bed
# Got 41936 genes.

# build hgIkmcExtra table

    cat 20110301_human.gff \
    | grep -v 'chrUn' \
    | perl -wpe 's/\r?\n$//; @w = split("\t"); \
      if ($w[3] eq "") { s/^.*//; next; } # Some lines have no coords. \
      if ($w[4] <= 0) { s/^.*//; next; } # A few lines have end=0. \
      $w[8] =~ m/^(MGI:\d+); (\w+); (\w.*)/ || die; \
      ($mgi, $designId, $status) = ($1, $2, $3); \
      $_ = "$w[10]_$designId\t$mgi,$designId,$w[2],$status\n";' \
    | sort -u > hgIkmcExtra.tab
    wc -l hgIkmcExtra.tab
# 41936 hgIkmcExtra.tab

# load tables
    hgLoadBed hg19 hgIkmc hgIkmc.bed
    checkTableCoords -verbose=2 hg19 hgIkmc

    hgLoadSqlTab hg19 hgIkmcExtra $HOME/kent/src/hg/lib/genericAlias.sql hgIkmcExtra.tab

#########################################################################
# LOAD ACEMBLY (DONE 2011-03-14 - Chin)
    mkdir /hive/data/outside/acembly
    cd /hive/data/outside/acembly
    wget --timestamping \
ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_37_Aug10.human.genes/AceView.ncbi_37.genes_gff.gff.gz
    wget --timestamping \
ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_37_Aug10.human.genes/AceView.ncbi_37.good_proteins_peptide.fasta.gz

    mkdir /cluster/data/hg19/bed/acembly
    cd /cluster/data/hg19/bed/acembly
    cp -p /hive/data/outside/acembly/AceView.ncbi_37.genes_gff.gff.gz .
    cp -p /hive/data/outside/acembly/AceView.ncbi_37.good_proteins_peptide.fasta.gz .
    gzip -d AceView.ncbi_37.genes_gff.gff.gz
    gzip -d AceView.ncbi_37.good_proteins_peptide.fasta.gz

    # If the result of this command is > 0, then some lines have end <
    # start
    # and need to be fixed:
    awk '$5 < $4 {print;}' AceView.ncbi_37.genes_gff.gff | wc -l
    # 0

    # Filter out empty lines, lines where the product_id has a stray
    # newline before it, and $chr|Hs# IDs that don't appear liftable.
    # (Note: the new gff does not have these two cases anymore.) 
    # Add 'chr' prefix to chrom number at field 1
    egrep -h -v '^(| ?product_id.*|..?\|Hs.*)$' AceView.ncbi_37.genes_gff.gff \
    | sed -e 's/^/chr/;' \
      > acembly.gff
    # fixed the chrmito prefix to chrM
    mv acembly.gff acembly.tmp
    cat acembly.tmp | sed -e 's/^chrmito/chrM/;' > acembly.gff 
    

    # Extract annotation classes from original gff:
    cat AceView.ncbi_37.genes_gff.gff | awk  '{print $12}' | sort | uniq
    # cDNA_supported;
    # Note: version 37 gff have only one gene type - cDNA_supported;
    # Per Danielle and Jean request, use pink to display them. 	
    # the following replace become no-op.
    egrep -h -v '^(| ?product_id.*|..?\|Hs.*)$' AceView.ncbi_37.genes_gff.gff \
    | perl -wpe 's/^.*Gene_type (\w+); transcript_id (\S+);.*/$2\t$1/; \
                 s/Cloud$/cloud/ || s/Spliced_gene$/spliced_gene/ || \
                   die "Unrecognized class/Gene_type:\n$_\n";' \
    | sort -u \
      > acemblyClass.tab
    # Useless use of single ref constructor in void context at -e line 2.
    # warning was issued, however it is harmless.


    # Some gff transcript_id's end in -unspliced (no intron), but the
    # corresponding protein fasta IDs to not have that suffix.  We need
    # them to match, so add where necessary.
    # Use perl to make a perl script to add -unspliced to protein IDs
    # where necessary:
    grep unspliced acemblyClass.tab | wc -l
    # 54180
    egrep -h -v '^(| ?product_id.*|..?\|Hs.*)$'  AceView.ncbi_37.genes_gff.gff \
    | perl -wpe 's@^.*transcript_id (\S+)-unspliced;.*$@\$unsp{"$1"} = 1;@ || s/^.*\n$//;' \
    | sort -u \
      > addUnspliced.pl
    wc -l addUnspliced.pl
    # 54180 addUnspliced.pl
    cat >> addUnspliced.pl <<'_EOF_'
while (<>) {
  if (/^>(\S+)$/) {
    if ($unsp{$1}) {
      s/^>(\S+)/>$1-unspliced/;
    }
  }
  print;
}
'_EOF_'
    # << emacs

    # Add -unspliced suffix to protein IDs where necessary, and pare
    # down
    # proteins to just the ones that we have transcripts for:
    awk '{print $1;}' acemblyClass.tab   > transcriptNames.txt
    perl addUnspliced.pl AceView.ncbi_37.good_proteins_peptide.fasta \
    | faSomeRecords stdin transcriptNames.txt acemblyPep.fa
    grep unspliced acemblyPep.fa | wc -l
    # 31956
    # Danielle Thierry-Mieg explained that noncoding genes are included
    # so
    # the number of proteins can be smaller than the number of
    # transcripts.

    # Load tables
    ssh hgwdev
    cd /cluster/data/hg19/bed/acembly
    ldHgGene -gtf hg19 acembly acembly.gff
    # Reading acembly.gff
    # Read 259440 transcripts in 3870073 lines in 1 files
    #   259440 groups 25 seqs 1 sources 5 feature types
    # 259440 gene predictions

    hgLoadSqlTab hg19 acemblyClass ~/kent/src/hg/lib/acemblyClass.sql \
      acemblyClass.tab
    # Scanning through 1 files
    hgPepPred hg19 generic acemblyPep acemblyPep.fa
    # rm acemblyPep.tab
    runJoiner.csh hg19 acembly
    # found identifiers:
    # acemblyName
    # Checking keys on database hg19
    #  hg19.acemblyPep.name - hits 187692 of 187692 ok
    #  hg19.acemblyClass.name - hits 259440 of 259440 ok
    # running -times flag


#############################################################################
# Affy Exon track (DONE 2011-03-08 - Melissa Cline)

# scripts/splitAffyExonBeds.py (below)
#!/usr/bin/env python

import fileinput
import re
import string

filehandle = None
for line in fileinput.input():
    if re.search("track\tname", line):
        if filehandle != None:
            filehandle.close()
        filename = fileinput.filename()
        filename = string.replace(filename, "hg19-bed", "hg19-split-bed")
        if re.search("gene level exon", line):
            filename = string.replace(filename, ".bed", ".exon.bed")
        elif re.search("gene probeset", line):
            filename = string.replace(filename, ".bed", ".geneProbeset.bed")
        elif re.search("exon probeset", line):
            filename = string.replace(filename, ".bed", ".probeset.bed")
        elif re.search("probe", line):
            filename = string.replace(filename, ".bed", ".probe.bed")
        filehandle = open(filename, 'w')
    filehandle.write(line)
        

# scripts/mergeAcrossChromosomes.bash (below)
#!/usr/bin/env bash


PATHNAME="/hive/users/cline/Affy/"
tail -n +2 $PATHNAME/Affy-HuEx-hg19-split-bed/HuEx-1_0-st-v2.hg19.*.$1.bed \
  |grep -v "==>" > $PATHNAME/mergedBeds/AffyHuEx.$1.bed 


# scripts/splitByProbesetType.py (below)
#!/usr/bin/env python
from optparse import OptionParser
import re

parser = OptionParser()
parser.add_option("-s", "--supplementalData", dest="supplementalData",
                  default="supportingAnnotations/HuEx-1_0-st-v2.na31.hg19.probeset.csv")
parser.add_option("-b", "--bedData", dest="bedData", default="noOverlaps/AffyHuEx.probeset.overlapsMerged.bed")
(parameters, args) = parser.parse_args()
coreProbesets = dict()
extendedProbesets = dict()
fullProbesets = dict()
ambiguousProbesets = dict()
freeProbesets = dict()
supplementalFile = open(parameters.supplementalData)
for line in supplementalFile:
    tokens = line.split(",")
    probesetId = re.sub("\"", "", tokens[0])
    if re.search("core", line):
        coreProbesets[probesetId] = 1
    elif re.search("extended", line):
        extendedProbesets[probesetId] = 1
    elif re.search("full", line):
        fullProbesets[probesetId] = 1
    elif re.search("ambiguous", line):
        ambiguousProbesets[probesetId] = 1
    elif re.search("free", line):
        freeProbesets[probesetId] = 1
supplementalFile.close()
coreProbesetsOutfile = re.sub("noOverlaps", "partitioned2",
                              re.sub("overlapsMerged", "core.overlapsMerged",
                                     parameters.bedData))
coreProbesetsOut = open(coreProbesetsOutfile, 'w')
extendedProbesetsOutfile = re.sub("noOverlaps", "partitioned2",
                                  re.sub("overlapsMerged",
                                         "extended.overlapsMerged",
                                         parameters.bedData))
extendedProbesetsOut = open(extendedProbesetsOutfile, 'w')
fullProbesetsOutfile = re.sub("noOverlaps", "partitioned2",
                              re.sub("overlapsMerged", "full.overlapsMerged",
                                     parameters.bedData))
fullProbesetsOut = open(fullProbesetsOutfile, 'w')
ambiguousProbesetsOutfile = re.sub("noOverlaps", "partitioned2",
                                   re.sub("overlapsMerged", "ambiguous.overlapsMerged",
                                          parameters.bedData))
ambiguousProbesetsOut = open(ambiguousProbesetsOutfile, 'w')
freeProbesetsOutfile = re.sub("noOverlaps", "partitioned2",
                                   re.sub("overlapsMerged", "free.overlapsMerged",
                                          parameters.bedData))
freeProbesetsOut = open(freeProbesetsOutfile, 'w')
bedInput = open(parameters.bedData)
for line in bedInput:
    tokens = line.split()
    if len(tokens) < 3:
        print "error: misformed line", line
    else:
        name = line.split()[3]
        (gene,probeset) = name.split("_")
        if coreProbesets.has_key(probeset):
            coreProbesetsOut.write(line)
        elif extendedProbesets.has_key(probeset):
            extendedProbesetsOut.write(line)
        elif fullProbesets.has_key(probeset):
            fullProbesetsOut.write(line)
        elif ambiguousProbesets.has_key(probeset):
            ambiguousProbesetsOut.write(line)
        elif freeProbesets.has_key(probeset):
            freeProbesetsOut.write(line)
        else:
            print "warning: orphan line", line


# 1. Given data from the vendor, one file per chromosome with four types of
# bed entries per file (one file per chromosome), split them into four bed 
# files (yielding four bed files per chromosome).
ls Affy-HuEx-hg19-bed/*bed \
    | awk '{ print "scripts/splitAffyExonBeds.py", $1 }' |bash
 
# 2. Given a subdirectory with four bed files per chromosome for probeset
# and probe bed files, merge them into one bed file for probeset data
# and one bed file per probe data.  From here, we are ignoring the exon
# and gene probeset data.
scripts/mergeAcrossChromosomes.bash probeset
scripts/mergeAcrossChromosomes.bash probe

# 3. There is an issue that the probeset data contains overlapping blocks.
# Fix this with bedMergeOverlappingBlocks, written by Andy Pohl.  Note that
# the probe data does not contain overlapping blocks.
bedMergeOverlappingBlocks mergedBeds/affyHuEx.probeset.bed \
    noOverlaps/AffyHuEx.probeset.overlapsMerged.bed
cp mergedBeds/AffyHuEx.probe.bed noOverlaps/AffyHuEx.probe.overlapsMerged.bed

# 4. There are five different types of probesets, each of which has a different
# significance to the user.  These should best be represented as five different
# subtracks of a parent track (one parent track for probesets, and one for probes).
# Split the data by probeset type.  Note that the same process can be applied to
# both the probeset and probe data, because the probe data is represented by the
# probeset ID rather than by a distinct probe ID.
scripts/splitByProbesetType.py \
    -s supportingAnnotations/HuEx-1_0-st-v2.na31.hg19.probeset.csv \
    -p noOverlaps/AffyHuEx.probe.overlapsMerged.bed
hgLoadBed hg19 affyExonProbeAmbiguous \
    partitioned2/AffyHuEx.probe.ambiguous.overlapsMerged.bed
hgLoadBed hg19 affyExonProbeCore \
    partitioned2/AffyHuEx.probe.core.overlapsMerged.bed
hgLoadBed hg19 affyExonProbeExtended \
    partitioned2/AffyHuEx.probe.extended.overlapsMerged.bed
hgLoadBed hg19 affyExonProbeFree \
    partitioned2/AffyHuEx.probe.free.overlapsMerged.bed
hgLoadBed hg19 affyExonProbeFull \
    partitioned2/AffyHuEx.probe.full.overlapsMerged.bed

scripts/splitByProbesetType.py \
    -s supportingAnnotations/HuEx-1_0-st-v2.na31.hg19.probeset.csv \
    -p noOverlaps/AffyHuEx.probeset.overlapsMerged.bed
hgLoadBed hg19 affyExonProbesetAmbiguous \
    partitioned2/AffyHuEx.probeset.ambiguous.overlapsMerged.bed
hgLoadBed hg19 affyExonProbesetCore \
    partitioned2/AffyHuEx.probeset.core.overlapsMerged.bed
hgLoadBed hg19 affyExonProbesetExtended \
    partitioned2/AffyHuEx.probeset.extended.overlapsMerged.bed
hgLoadBed hg19 affyExonProbesetFree \
    partitioned2/AffyHuEx.probeset.free.overlapsMerged.bed
hgLoadBed hg19 affyExonProbesetFull \
    partitioned2/AffyHuEx.probeset.full.overlapsMerged.bed


#########################################################################
# LASTZ Turkey MelGal1 (DONE - 2011-03-28 - Chin)
    mkdir /hive/data/genomes/hg19/bed/lastzMelGal1.2011-03-28
    cd /hive/data/genomes/hg19/bed/lastzMelGal1.2011-03-28

    cat << '_EOF_' > DEF
# Turkey vs Human

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Turkey melGal1 - single chunk big enough to run entire chrom
SEQ2_DIR=/scratch/data/melGal1/melGal1.2bit
SEQ2_LEN=/scratch/data/melGal1/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzMelGal1.2011-03-28
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
        `pwd`/DEF \
        -syntenicNet \
        -noLoadChainSplit \
        -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
        -chainMinScore=5000 -chainLinearGap=loose > do.log 2>&1
    # real    100m33.646s
    cat fb.hg19.chainMelGal1Link.txt
    #   76647912 bases of 2897316137 (2.645%) in intersection
    # Create link
    cd /hive/data/genomes/hg19/bed
    ln -s  lastzMelGal1.2011-03-28 lastz.melGal1


    #   running the swap 
    mkdir /hive/data/genomes/melGal1/bed/blastz.hg19.swap
    cd /hive/data/genomes/melGal1/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
        /hive/data/genomes/hg19/bed/lastzMelGal1.2011-03-28/DEF \
        -swap \
        -noLoadChainSplit \
        -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
        -chainMinScore=5000 -chainLinearGap=loose > swap.log 2>&1
    #   real    6m51.280s
    cat fb.melGal1.chainHg19Link.txt
    #   62120143 bases of 935922386 (6.637%) in intersection
    cd /hive/data/genomes/melGal1/bed
    ln -s blastz.hg19.swap lastz.hg19

#############################################################################
# BUILD seudoYale60 TRACK 
# Rachel built this for Gencode first, Fan adopted for UCSC GB, DONE, 3/31/2011.
#
# First, how Rachel built it:
# YALE PSEUDOPIPE PSEUDOGENE PREDICTIONS BASED ON ENSEMBL 60 
# (hartera, 2011-01-10, DONE)
# FTP site e-mailed by Suganthi Balasubramanian (suganthi.bala@yale.edu) from 
# the Gerstein lab. Data is from their PseudoPipe pipeline and it is based on 
# proteins from Ensembl Build 60 (pseudogene data from December 2010?). 

mkdir -p /hive/groups/gencode/browser/hg19/gencodeYalePseudoBuild60
cd /hive/groups/gencode/browser/hg19/gencodeYalePseudoBuild60

wget --timestamping \
     "http://tables.pseudogene.org/dump.cgi?table=Human60"

# Then re-name the file:
mv dump.cgi\?table=Human60 Human60YalePseudo.txt
# Header from data file.
ID      Chromosome      Start Coordinate        Stop Coordinate Strand  Parent
Protein  Protein Start   Protein Stop    Parent Gene     Fraction        Num
Insertions  Num Deletions   Num Shifts      Num Stops       E Value Identity
PolyA   Disablements    Exons   Introns Class   Sequence        Link

# urls are of type:
# http://tables.pseudogene.org/human60/<ID> so this can be added to the
# trackDb as for the previous track. 

# Get list of haplotype chroms:
grep _ Human60YalePseudo.txt | awk '{print $2}' | sort | uniq
HSCHR17_1
HSCHR6_MHC_APD
HSCHR6_MHC_COX
HSCHR6_MHC_DBB
HSCHR6_MHC_MANN
HSCHR6_MHC_MCF
HSCHR6_MHC_QBL
HSCHR6_MHC_SSTO

# These correspond to the haplotype chroms in GRCh37 (hg19).
# Convert data to genePred:
# chromsomomes are 1-22, X, Y, chr17_ctg5_hap1 (HSCHR17_1) and the chr6 
# haplotypes e.g. chr6_cox_hap1 (HSCHR6_MHC_COX) 

cat << '_EOF_' > formatPseudogenesToGenePred
#!/usr/bin/awk -f
# Parse Yale pseudogene data file.
# Exon coordinates are in this format: [[28688544, 28688864], [28689678, 2869117# 4], [28694308, 28694460], [28701327, 28701749]]
# Ignore header line
/^ID/ {
  next;
}
# Parse the data lines
BEGIN {FS="\t"} {OFS="\t"} {
  gsub(/\[/, "", $19);
  gsub(/\]/, "", $19); 
  split($19, exons, ",");
  # Count the number of start and end coordinates for exons and 
  # calculate the number of exons.
  count=(length(exons))/2;
  # Write out genePred. Add chr in front of chrom only if not haplotype.
  if ($2 !~ /HSCHR/) {
     printf "%s\tchr%s\t%c\t%d\t%d\t0\t0\t%d\t", $1, $2, $5, $3-1, $4, count; 
  }
  else {
     printf "%s\t%s\t%c\t%d\t%d\t0\t0\t%d\t", $1, $2, $5, $3-1, $4, count; 
  }
  # get list of exon starts, convert from 1-based to 0-based
  for (i=1; i <= length(exons); i+=2) {
     printf "%d,", exons[i]-1","; 
  }
  printf "\t";
  # get list of exon ends
  for (i=2; i <= length(exons); i+=2) {
     printf "%d,", exons[i]","; 
  }
  printf "\n"; 
}
'_EOF_'

chmod +x formatPseudogenesToGenePred
# format the Yale pseudogenes data to genePred.
./formatPseudogenesToGenePred Human60YalePseudo.txt \
    > gencodeYalePseudoBuild60.gp 

# The Genome Browser represents just the haplotype region as a separate 
# "chromosome" whereas the coordinates represent the haplotype region embedded
# into chr6.
cp -p /hive/data/genomes/hg19/jkStuff/ensGene.haplotype.lift .
# The lift file assumes the following chrom names 
-69170076       HSCHR4_1        191154276       chr4_ctg9_hap1  590426
-43384863       HSCHR17_1       81195210        chr17_ctg5_hap1 1680828
-28696603       HSCHR6_MHC_APD  171115067       chr6_apd_hap1   4622290
-28477796       HSCHR6_MHC_COX  171115067       chr6_cox_hap2   4795371
-28696603       HSCHR6_MHC_DBB  171115067       chr6_dbb_hap3   4610396
-28696603       HSCHR6_MHC_MANN 171115067       chr6_mann_hap4  4683263
-28696603       HSCHR6_MHC_MCF  171115067       chr6_mcf_hap5   4833398
-28696603       HSCHR6_MHC_QBL  171115067       chr6_qbl_hap6   4611984
-28659142       HSCHR6_MHC_SSTO 171115067       chr6_ssto_hap7  4928567

liftUp -type=.gp gencodeYalePseudoBuild60HapsLifted.gp \
       ensGene.haplotype.lift carry gencodeYalePseudoBuild60.gp
# Got 68 lifts in ensGene.haplotype.lift
# Lifting gencodeYalePseudoBuild55.gp

wc -l gencode*.gp
17888 gencodeYalePseudoBuild60.gp
17888 gencodeYalePseudoBuild60HapsLifted.gp
 
# Load table and then check some haplotype regions. See if look plausible. 
# Load the genePred file into hg19
hgLoadGenePred hg19 gencodeYalePseudoBuild60 \
gencodeYalePseudoBuild60HapsLifted.gp
# Didn't load. There are 12 invalid genePreds (10 in Ensembl 55, 12 for 59):
Error: invalid genePred: PGOHUM00000244617 exon 2 overlaps previous exon
Error: invalid genePred: PGOHUM00000244796 exon 1 overlaps previous exon
Error: invalid genePred: PGOHUM00000248470 exon 1 overlaps previous exon
Error: invalid genePred: PGOHUM00000251325 exon 1 overlaps previous exon
Error: invalid genePred: PGOHUM00000250199 exon 1 overlaps previous exon
Error: invalid genePred: PGOHUM00000243651 exon 1 overlaps previous exon
Error: invalid genePred: PGOHUM00000232858 exon 1 overlaps previous exon
Error: invalid genePred: PGOHUM00000232933 exon 1 overlaps previous exon
Error: invalid genePred: PGOHUM00000233065 exon 1 overlaps previous exon
Error: invalid genePred: PGOHUM00000236237 exon 1 overlaps previous exon
Error: invalid genePred: PGOHUM00000241760 exon 1 overlaps previous exon
Error: invalid genePred: PGOHUM00000233784 exon 8 overlaps previous exon
Error: 12 invalid genePreds, database unchanged

# These are on chroms 1, 6, 7, 9, X, Y.
# File didn't load into database.
# Make a file of these ids - invalidIds
grep -f invalidIds -vw gencodeYalePseudoBuild60HapsLifted.gp \
    > gencodeYalePseudoBuild60HapsLiftedNoInvalidGps.gp
wc -l gencode*gp
# 17888 gencodeYalePseudoBuild60.gp
# 17888 gencodeYalePseudoBuild60HapsLifted.gp
# 17876 gencodeYalePseudoBuild60HapsLiftedNoInvalidGps.gp

# Then re-load database
hgLoadGenePred hg19 gencodeYalePseudoBuild60 \
      gencodeYalePseudoBuild60HapsLiftedNoInvalidGps.gp

# Add trackDb.ra entry for track, add a search and make sure
# there is a description page, copy over from the gencodeYalePseudoBuild59
# html. Commit these to SVN. 
# Add to the html description, the list of 12 IDs of genes that were removed 
# due to overlapping exon coordinates. This was also a problem for the Yale
# pseudogenes based on Ensembl Build 53 and 55 but there were 10 problem IDs
# for those builds. 

# Build class table for colouring pseudogenes by type. 
# copy over class table definition from a previous set of Yale pseudogenes.
cp -p ../gencodeYalePseudoBuild55/gencodeYalePseudoBuild55Class.sql \
 gencodeYalePseudoBuild60Class.sql

# Make the class table file:
tail -n +2 Human60YalePseudo.txt \
  | tawk '{print $1, $21, "Yale"}' | sort > yalePseudoBuild60Class.txt

# load table
hgLoadSqlTab hg19 gencodeYalePseudoBuild60Class \
    gencodeYalePseudoBuild60Class.sql yalePseudoBuild60Class.txt

hgsql -e 'select distinct(class) from gencodeYalePseudoBuild60Class;' hg19
+------------+
| class      |
+------------+
| Ambiguous  | 
| Processed  | 
| Duplicated | 
+------------+

# Add these classes to the trackDb.ra entry for the geneClasses field and 
# to the list of classes with colours.

# Next, how Fan adopted it:

ssh hgwdev
mkdir -p /hive/data/genomes/hg19/bed/pseudoYale60
cd /hive/data/genomes/hg19/bed/pseudoYale60

hgsql hg19 < ~/src/hg/lib/pseudoYale60.sql
hgsql hg19 < ~/src/hg/lib/pseudoYale60Class.sql

hgsql hg19 -e "insert into pseudoYale60 select * from gencodeYalePseudoBuild60"
hgsql hg19 -e "insert into pseudoYale60Class select * from gencodeYalePseudoBuild60Class"

##############################################################################
# LASTZ Lizard AnoCar2 (DONE - 2011-04-19 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzAnoCar2.2011-04-19
    cd /hive/data/genomes/hg19/bed/lastzAnoCar2.2011-04-19

    cat << '_EOF_' > DEF
# human vs lizard
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Lizard anoCar2
SEQ2_DIR=/scratch/data/anoCar2/anoCar2.2bit
SEQ2_LEN=/scratch/data/anoCar2/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0
SEQ2_LIMIT=40

BASE=/hive/data/genomes/hg19/bed/lastzAnoCar2.2011-04-19
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-syntenicNet -workhorse=hgwdev -smallClusterHub=encodek \
	-bigClusterHub=swarm -qRepeats=windowmaskerSdust > do.log 2>&1 &
    #	real    195m52.809s
    cat fb.hg19.chainAnoCar2Link.txt 
    #	102917023 bases of 2897316137 (3.552%) in intersection

    #	running the swap - DONE - 2011-04-19
    mkdir /hive/data/genomes/anoCar2/bed/blastz.hg19.swap
    cd /hive/data/genomes/anoCar2/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzAnoCar2.2011-04-19/DEF \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	-syntenicNet -swap -qRepeats=windowmaskerSdust > swap.log 2>&1 &
    #	real    20m45.683s
    cat fb.anoCar2.chainHg19Link.txt 
    #	88296392 bases of 1701353770 (5.190%) in intersection

##############################################################################
# NCBI patch 3 (NOT COMPLETE - 2011-04-21 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/additionalSequence/patch3
    cd /hive/data/genomes/hg19/bed/additionalSequence/patch3
    # these scripts were altered slightly for improvements and corrections
    # to this patch3 business
    cp -p ../patches/gatherNames.pl .
    # business added to gatherNames.pl to construct patches.chrom.sizes file
    ./gatherNames.pl . > ucscNames.patch3.txt
    cp -p ../patch2/mkTables.pl .
    ./mkTables.pl  patches.chrom.sizes ucscNames.patch3.txt PATCHES/alt_scaffolds/AGP/alt.scaf.agp.gz
    # output to stdout is the contents of alt.scaf.agp.gz
    # constructs files: ctgPos.txt chromInfo.txt gold.txt and gap.txt
    cp -p ../patch2/mkCtgPos2.pl .
    ./mkCtgPos2.pl ucscNames.patch3.txt patches.chrom.sizes > ctgPos2.txt
    cp -p ../patch2/mkHapLocate.pl .
    ./mkHapLocate.pl ctgPos.txt \
	PATCHES/alt_scaffolds/alt_scaffold_placement.txt \
	> haplotypeLocations.bed
# not found: GL339449.1 HSCHR5_1_CTG1
# not found: GL339450.1 HG79_PATCH

##############################################################################
# NCBI patch 5 (DONE - 2011-07-01,13 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/additionalSequence/patch5
    cd /hive/data/genomes/hg19/bed/additionalSequence/patch5
    wget --cut-dirs=6 --no-parent --timestamping --no-remove-listing -m \
        -nH --ftp-user=anonymous --ftp-password=yourName@domain.edu \
ftp://ftp.ncbi.nlm.nih.gov/genbank/genomes/Eukaryotes/vertebrates_mammals/Homo_sapiens/GRCh37.p5/
    # the scripts from patch4 were modified slightly to update and fix some
    #	of the new names in this patch5
    cp ../patch4/gatherNames.pl .
    ./gatherNames.pl . > ucscNames.patch5.txt
    cp -p ../patch4/mkTables.pl .
    ./mkTables.pl  patches.chrom.sizes ucscNames.patch5.txt PATCHES/alt_scaffolds/AGP/alt.scaf.agp.gz
    # output to stdout is the contents of alt.scaf.agp.gz
    # constructs files: ctgPos.txt chromInfo.txt gold.txt and gap.txt
    cp -p ../patch4/mkCtgPos2.pl .
    ./mkCtgPos2.pl ucscNames.patch5.txt patches.chrom.sizes > ctgPos2.txt
    cp -p ../patch4/mkHapLocate.pl .
    ./mkHapLocate.pl ctgPos.txt \
	PATCHES/alt_scaffolds/alt_scaffold_placement.txt \
	> haplotypeLocations.bed
    cp haplotypeLocations.bed altSequence.bed
    ln -s ../patch2/before.patch2.hapLoc.bed hg19.hapLoc.bed
    awk '{printf "%s\t%d\t%d\t%s\t500\t+\t%d\t%d\t32,32,190\n", $2,$3,$4,$5,$3,$4}' \
hg19.hapLoc.bed >> altSequence.bed

    # a new script for patch5
    ./mkFasta.pl ucscNames.patch5.txt > hg19.patch5.fa
    # the build of hg19Patch5 can be seen in hg19Patch5.txt

    egrep -v "32,32,190" altSequence.bed  \
	| awk '{printf "%s\t%d\t%d\t%s\t%d\t%s\n", $1,$2,$3,$4,$5,$6}' \
	    > altSeqPatchesP5.tab
    egrep "32,32,190" altSequence.bed  \
	| awk '{printf "%s\t%d\t%d\t%s\t%d\t%s\n", $1,$2,$3,$4,$5,$6}' \
	    > altSeqHaplotypesP5.tab
    # verify none lost
    wc -l altSeqPatchesP5.tab altSeqHaplotypesP5.tab
    #	41 altSeqPatchesP5.tab
    #	75 altSeqHaplotypesP5.tab
    #	116 total
    wc -l altSequence.bed
    #	116 altSequence.bed
    hgLoadBed hg19 altSeqHaplotypesP5 altSeqHaplotypesP5.tab
    #	Loaded 75 elements of size 6
    hgLoadBed hg19 altSeqPatchesP5 altSeqPatchesP5.tab
    #	Loaded 41 elements of size 6

    # to replace the existing track:
    hgLoadBed hg19 altSeqHaplotypes altSeqHaplotypesP5.tab
    #	Loaded 75 elements of size 6
    hgLoadBed hg19 altSeqPatches altSeqPatchesP5.tab
    #	Loaded 41 elements of size 6
    
##############################################################################
#  hg19 - Human - Ensembl Genes version 62  (DONE - 2011-04-22 - hiram)
    # This human gene set need a lot of work to get the name translation
    #	to work again.  The contig names have changed in Ensembl for this
    #	version and they defined genes on patch sequence that UCSC does not
    #	include
    ssh hgwdev
    cd /hive/data/genomes/hg19
    cat << '_EOF_' > hg19.ensGene.ra
# required db variable
db hg19
# optional nameTranslation, the sed command that will transform
#       Ensemble names to UCSC names.  With quotes just to make sure.
# delete commands take out genes that are only in patch sequence
nameTranslation 's/^\([0-9XY][0-9]*\)/chr\1/; s/^MT/chrM/; /^GL3.*/d; /^HSCHR[1-5]/d; /^HSCHR[7-9]/d; /^HG/d'
# optionally update the knownToEnsembl table after ensGene updated
knownToEnsembl yes
# optional haplotype lift-down from Ensembl full chrom coordinates
#       to UCSC simple haplotype coordinates
haplotypeLift /hive/data/genomes/hg19/jkStuff/ensGene.haplotype.lift
# Ensembl 62 has new sequence names for some of the random bits
liftUp /hive/data/genomes/hg19/jkStuff/ens.62.lft
'_EOF_'
#  << happy emacs

    doEnsGeneUpdate.pl  -ensVersion=62 hg19.ensGene.ra
    ssh hgwdev
    cd /hive/data/genomes/hg19/bed/ensGene.62
    featureBits hg19 ensGene
    # 109947258 bases of 2897316137 (3.795%) in intersection

    hgsql -e \
'update trackVersion set dateReference="current" where db="hg19" AND version=62;' hgFixed

############################################################################
# BUILD hg19 GERP TRACK (DONE 4/25/11, Fan)

ssh hgwdev
mkdir /hive/data/genomes/hg19/bed/gerp
cd /hive/data/genomes/hg19/bed/gerp

# place the wig data file, All_hg19_RS.wig, here.

ulimit -d 180000000
ulimit -v 180000000

wigToBigWig All_hg19_RS.wig /hive/data/genomes/hg19/chrom.sizes All_hg19_RS.bw

ln -s `pwd`/All_hg19_RS.bw /gbdb/hg19/bbi/All_hg19_RS.bw

hgsql hg19 -e 'drop table if exists allHg19RS_BW; \
               create table allHg19RS_BW (fileName varchar(255) not null); \
	       insert into allHg19RS_BW values ("/gbdb/hg19/bbi/All_hg19_RS.bw");'

# create corresponding trackDb.ra section and html description page.


#########################################################################
# LASTZ Cow BosTau6 (DONE - 2011-05-16 - Chin)
    mkdir /hive/data/genomes/hg19/bed/lastzBosTau6.2011-05-16
    cd /hive/data/genomes/hg19/bed/lastzBosTau6.2011-05-16

    cat << '_EOF_' > DEF
# human vs cow
# maximum M allowed with lastz is only 254
BLASTZ_M=254

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Cow bosTau6 
SEQ2_DIR=/scratch/data/bosTau6/bosTau6.2bit
SEQ2_LEN=/scratch/data/bosTau6/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/hive/data/genomes/hg19/bed/lastzBosTau6.2011-05-16
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
        `pwd`/DEF \
        -syntenicNet \
        -noLoadChainSplit \
        -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
        -chainMinScore=3000 -chainLinearGap=medium > do.log 2>&1
    # real    481m23.263s 
    cat fb.hg19.chainBosTau6Link.txt
    # 1370696434 bases of 2897316137 (47.309%) in intersection
    # Create link
    cd /hive/data/genomes/hg19/bed
    ln -s  lastzBosTau6.2011-05-16 lastz.bosTau6


    #   running the swap 
    mkdir /hive/data/genomes/bosTau6/bed/blastz.hg19.swap
    cd /hive/data/genomes/bosTau6/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
        /hive/data/genomes/hg19/bed/lastzBosTau6.2011-05-16/DEF \
        -swap  -syntenicNet \
        -noLoadChainSplit \
        -workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
        -chainMinScore=3000 -chainLinearGap=medium > swap.log 2>&1
    #   real     98m22.477s
    cat fb.bosTau6.chainHg19Link.txt
    #   1336966333 bases of 2649682029 (50.458%) in intersection
    cd /hive/data/genomes/bosTau6/bed
    ln -s blastz.hg19.swap lastz.hg19


###################################################################
# BUILD OMIM RELATED TRACKS (REBUILT 5/17/11, Fan)

ssh hgwdev
cd /hive/data/genomes/hg19/bed
mkdir -p omim/05172011
cd omim/05172011

# obtain the following files from OMIM and place them at this subdirectory

       genemap.txt  
       mim2gene.txt  
       mimAV.txt
       script1.pl  
       script2.pl

cat genemap.txt|sed -e 's/|/\t/g' > genemap.tab

hgLoadSqlTab -warn hg19 omimGeneMap ~/kent/src/hg/lib/omimGeneMap.sql genemap.tab

# Load mim2gene table

hgsql hg19 -e 'drop table mim2gene'
hgsql hg19 < ~/kent/src/hg/lib/mim2gene.sql
hgsql hg19 -e 'load data local infile "mim2gene.txt" into table mim2gene ignore 1 lines'

doOmimDisorders hg19 omimDisorderMap.tab
hgsql hg19 -e "drop table omimDisorderMap"
hgsql hg19 < ~/kent/src/hg/lib/omimDisorderMap.sql
hgLoadSqlTab -warn hg19 omimDisorderMap ~/kent/src/hg/lib/omimDisorderMap.sql omimDisorderMap.tab 

# build omimGeneSymbol table

doOmimGeneSymbols hg19 j.out
cat j.out |sort -u >omimGeneSymbol.tab
hgLoadSqlTab -warn hg19 omimGeneSymbol ~/kent/src/hg/lib/omimGeneSymbol.sql omimGeneSymbol.tab 

perl ./script1.pl --gene-map-file=genemap.txt >omimPhenotype.tab
hgLoadSqlTab -warn hg19 omimPhenotype ~/kent/src/hg/lib/omimPhenotype.sql omimPhenotype.tab 

hgsql hg19 -e 'update omimPhenotype set phenotypeClass = -1 where
phenotypeClass=0'
hgsql hg19 -e 'update omimPhenotype set phenotypeId = -1 where phenotypeId=0'

doOmimGene2 hg19 j.tmp
cat j.tmp |sort -u > omimGene2.tab

hgLoadBed hg19 omimGene2 omimGene2.tab

rm j.tmp
##############################################################
# build the omimAvSnp track

cd /hive/data/genomes/hg19/bed/omim/05172011
mkdir av
cd av

# get the mimAV.txt data file from OMIM

cut -f 1 mimAV.txt >j1
cut -f 2 mimAV.txt >j2
cut -f 3  mimAV.txt >j3
cut -f 4  mimAV.txt >j4
cut -f 5  mimAV.txt >j5

cat j1 |sed -e 's/\./\t/' >j1.2

cat j4 |sed -e 's/,/\t/' >j4-2
cut -f 1 j4-2 >j4.1
cut -f 2 j4-2 >j4.2

paste j1 j1.2 j3 j4 j4.1 j4.2 j5 j2 >omimAv.tab

hgsql hg19 -e 'drop table omimAv'
hgsql hg19 < ~/src/hg/lib/omimAv.sql
hgsql hg19 -e \
'load data local infile "omimAv.tab" into table omimAv ignore 1 lines'
hgsql hg19 -e 'update omimAv set repl2 = rtrim(ltrim(repl2))'

doOmimAv hg19 omimAvRepl.tab  2>j.err

hgsql hg19 -e "drop table omimAvRepl"
hgsql hg19 < ~/kent/src/hg/lib/omimAvRepl.sql
hgsql hg19 -e 'load data local infile "omimAvRepl.tab" into table omimAvRepl'

rm j1.2  j1 j2 j3  j4  j4-2  j4.1  j4.2  j5

hgsql hg19 -N -e 'select chrom, chromStart, chromEnd, avId from omimAvRepl r,
snp132 s where s.name = dbSnpId order by avId' >omimAvSnp.tab

hgLoadBed -allowStartEqualEnd  hg19 omimAvSnp omimAvSnp.tab
##############################################################
# build the omimLocation track

cd /hive/data/genomes/hg19/bed/omim/05172011
mkdir location
cd location

doOmimLocation hg19 omimLocation.bed 2>j.err
hgLoadBed hg19 omimLocation omimLocation.bed

# Remove all gene entries in omimGene2 from omimLocation table

hgsql hg19 -N -e \
'delete from omimLocation where name  in (select name from omimGene2) '

# Per OMIM request, delete all the gray entries in omimLocation table.

mkdir cleanUpOmimLocation
cd cleanUpOmimLocation

hgsql hg19 -N -e \
'select distinct name from omimLocation' |sort -u >j.all

hgsql hg19 -N -e \
'select distinct name from omimLocation, omimPhenotype where name=omimId and phenotypeClass=1' >j.1
hgsql hg19 -N -e \
'select distinct name from omimLocation, omimPhenotype where name=omimId and phenotypeClass=2' >j.2
hgsql hg19 -N -e \
'select distinct name from omimLocation, omimPhenotype where name=omimId and phenotypeClass=3' >j.3
hgsql hg19 -N -e \
'select distinct name from omimLocation, omimPhenotype where name=omimId and phenotypeClass=4' >j.4

cat j.1 j.2 j.3 j.4 |sort -u >j.1234

diff j.all j.1234 |grep "<" |sed -e 's/</do1/' >doall

cat << '_EOF_' > do1
hgsql hg19 -e "delete from omimLocation where name='${1}'"
'_EOF_'
# << emacs

./doall

############################################################################
# NUMTS TRACK (DONE 2011-06-03 - Chin)

    mkdir -p /hive/data/outside/Numts/hg19
    cd /hive/data/outside/Numts/hg19
    wget http://193.204.182.50/files/hg19/all_hg19_NumtS_tracks.txt
    wget http://193.204.182.50/files/hg19/HSA_NumtS_hg19_details.html
    wget http://193.204.182.50/files/bam/allNumtS.sorted.bam
    wget http://193.204.182.50/files/bam/allNumtS.sorted.bam.bai

    mkdir /cluster/data/hg19/bed/NumtS
    cd  /cluster/data/hg19/bed/NumtS
    cp /hive/data/outside/Numts/hg19/*.* .


    # split the all_hg19_NumtS_tracks.txt into 3 bed files 
    # numtSAssembled.bed, numtS.bed, an numtSMitochondrion.bed

    cat all_hg19_NumtS_tracks.txt | awk ' /^track name/ {print $_}'  > tracks.list
    cat all_hg19_NumtS_tracks.txt | awk ' /^track type/ {print $_}'  >> tracks.list

    # load the 3 bed files to hg19
    hgLoadBed hg19  numtSAssembled  numtSAssembled.bed
    hgLoadBed hg19 numtS numtS.bed
    hgLoadBed hg19 numtSMitochondrion numtSMitochondrion.bed
    # Make /gbdb/ links and load bam
    mkdir /gbdb/hg19/NumtS
    ln -s `pwd`/allNumtS.sorted.bam{,.bai} /gbdb/hg19/NumtS/
    hgBbiDbLink hg19 bamAllNumtSSorted /gbdb/hg19/NumtS/allNumtS.sorted.bam 

    # setup trackDb for hg19

############################################################################
# Add Gene name search to Ensembl gene track (DONE - 2011-07-22 - Hiram)
    cd /hive/data/genomes/hg19/bed/ensGene.62/process
    cut -f1,9 infoOut.txt | grep -v "^#" | sort > ensemblToGeneName.tab
    NL=`awk '{print length($1)}' ensemblToGeneName.tab | sort -rn | head -1`
    VL=`awk '{print length($2)}' ensemblToGeneName.tab | sort -rn | head -1`
    sed -e "s/ knownTo / ensemblToGeneName /; s/known gene/ensGen/; s/INDEX(name(12)/PRIMARY KEY(name($NL)/; s/value(12)/value($VL)/" \
	$HOME/kent/src/hg/lib/knownTo.sql > ensemblToGeneName.sql

    hgLoadSqlTab hg19 ensemblToGeneName ensemblToGeneName.sql ensemblToGeneName.tab

    # add this search specification to trackDb.ra
searchName ensGeneName
searchTable ensGene
searchType genePred
searchMethod prefix
xrefTable ensemblToGeneName
xrefQuery select name,value from %s where value like '%%%s%%'
searchPriority 50

############################################################################
# COSMIC TRACK (DONE 2011-07-15 Fan)

mkdir /hive/data/outside/cosmic/20110711
# put raw data file, EnsMutExp_v54_080711.csv, received by email to there.

mkdir /hive/data/genomes/hg19/bed/cosmic/20110711
cd /hive/data/genomes/hg19/bed/cosmic/20110711

cp -p /hive/data/outside/cosmic/20110711/EnsMutExp_v54_080711.csv . 

cat EnsMutExp_v54_080711.csv|sed -e 's/\t//g' |sed -e 's/,/\t/g' |\
grep -v COSMIC_MUTATION_ID |grep -v 'selected'|grep COSM
>EnsMutExp_v54_080711.tab

hgsql hg19 -e 'drop table cosmicRaw'
hgsql hg19 < ~/kent/src/hg/lib/cosmicRaw.sql

hgLoadSqlTab hg19 cosmicRaw ~/kent/src/hg/lib/cosmicRaw.sql
EnsMutExp_v54_080711.tab

# use  grch37_start-1 for our zero based chromStart and 
# conver their chr23 and chr24 to chrX and chrY.

hgsql hg19 -N -e 'select "chr", chromosome, grch37_start-1, grch37_stop,
cosmic_mutation_id from cosmicRaw' \
|grep -v NULL |sed -e 's/chr\t/chr/'|sort -u|sed -e 's/chr23/chrX/' |sed -e
's/chr24/chrY/' >cosmic.bed

hgLoadBed -allowStartEqualEnd  hg19 cosmic cosmic.bed

#############################################################################
# HI SEQ DEPTH (DONE 7/15/11 angie)
    mkdir /hive/data/genomes/hg19/bed/hiSeqDepth
    cd /hive/data/genomes/hg19/bed/hiSeqDepth
    foreach cov (001 005 01 05 1)
      wget --timestamp http://eqtl.uchicago.edu/Masking/seq.cov$cov.ONHG19.bed.gz
      gunzip -N seq.cov$cov.ONHG19.bed.gz
    end
    wc -l seq.cov*
#    522 seq.cov001.ONHG19.bed
#   1224 seq.cov005.ONHG19.bed
#   2060 seq.cov01.ONHG19.bed
#  16119 seq.cov05.ONHG19.bed
#  30671 seq.cov1.ONHG19.bed
    foreach cov (001 005 01 05 1)
      echo seq.cov$cov.ONHG19.bed
      featureBits -countGaps hg19 seq.cov$cov.ONHG19.bed
    end
#seq.cov001.ONHG19.bed
#55092 bases of 3137161264 (0.002%) in intersection
#seq.cov005.ONHG19.bed
#175379 bases of 3137161264 (0.006%) in intersection
#seq.cov01.ONHG19.bed
#344425 bases of 3137161264 (0.011%) in intersection
#seq.cov05.ONHG19.bed
#3073270 bases of 3137161264 (0.098%) in intersection
#seq.cov1.ONHG19.bed
#5736695 bases of 3137161264 (0.183%) in intersection
    # Compare hg19 coverage to hg18:
    calc 55092 / 57409
#55092 / 57409 = 0.959640
    calc 175379 / 183848
#175379 / 183848 = 0.953935
    calc 344425 / 362423
#344425 / 362423 = 0.950340
    calc 3073270 / 3462959
#3073270 / 3462959 = 0.887469
    calc 5736695 / 6466376
#5736695 / 6466376 = 0.887158

    # Not all small ones are strict subsets of larger ones.
    featureBits hg19 -countGaps seq.cov001.ONHG19.bed \!seq.cov005.ONHG19.bed
#128 bases of 3137161264 (0.000%) in intersection
    featureBits hg19 -countGaps seq.cov005.ONHG19.bed \!seq.cov01.ONHG19.bed
#222 bases of 3137161264 (0.000%) in intersection
    featureBits hg19 -countGaps seq.cov01.ONHG19.bed \!seq.cov05.ONHG19.bed
#4185 bases of 3137161264 (0.000%) in intersection
    featureBits hg19 -countGaps seq.cov05.ONHG19.bed \!seq.cov1.ONHG19.bed
#41831 bases of 3137161264 (0.001%) in intersection
    # No overlap w/gap track:
    featureBits hg19 -countGaps seq.cov1.ONHG19.bed gap -bed=gapOverlaps.ONHG19.bed
#0 bases of 3137161264 (0.000%) in intersection

    # Load tables:
    hgLoadBed hg19 hiSeqDepthTopPt1Pct seq.cov001.ONHG19.bed
#Loaded 522 elements of size 3
    hgLoadBed hg19 hiSeqDepthTopPt5Pct seq.cov005.ONHG19.bed
#Loaded 1224 elements of size 3
    hgLoadBed hg19 hiSeqDepthTop1Pct seq.cov01.ONHG19.bed
#Loaded 2060 elements of size 3
    hgLoadBed hg19 hiSeqDepthTop5Pct seq.cov05.ONHG19.bed
#Loaded 16119 elements of size 3
    hgLoadBed hg19 hiSeqDepthTop10Pct seq.cov1.ONHG19.bed
#Loaded 30671 elements of size 3

############################################################################
# adding new decode data (DONE - 2011-08-18 - Hiram)
# liftOver from hg18 tracks:
    mkdir /hive/data/outside/decode/hg19
    cd /hive/data/outside/decode/hg19
# some of the items end up overlapping as result of liftOver,
# this filters them out:
export OVERLAPS="9202536|9235404|9225403|9215395|9192536|9182536|9172536|110561813|110582154|110572154|110552191|110542194|110532192|110522233|110512223|110502226|110492220|110482221|110472216|36283195|36273203|36252392|81325902|36262322"

for T in female female_carrier female_noncarrier male male_carrier \
        male_noncarrier sex-averaged sex-averaged_carrier \
        sex-averaged_noncarrier maleFemale
do
liftOver ../hg18/${T}.bedGraph \
        /hive/data/genomes/hg18/bed/liftOver/hg18ToHg19.over.chain.gz \
        ${T}.hg19.txt ${T}.unMapped.bedGraph
wc -l ${T}.hg19.txt ${T}.unMapped.bedGraph
awk '$3-$2 > 8000 && $3-$2 < 12000' ${T}.hg19.txt | sort -k1,1 -k2,2n \
	| egrep -v "${OVERLAPS}" > ${T}.hg19.bedGraph
awk '$3-$2 < 8001 || $3-$2 > 11999' ${T}.hg19.txt >> ${T}.unMapped.bedGraph
awk '$3-$2 > 8000 && $3-$2 < 12000' ${T}.hg19.txt | sort -k1,1 -k2,2n \
    | egrep "${OVERLAPS}" >> ${T}.unMapped.bedGraph
bedGraphToBigWig ${T}.hg19.bedGraph /hive/data/genomes/hg19/chrom.sizes ${T}.bw
wc -l ${T}.hg19.txt ${T}.hg19.bedGraph ${T}.unMapped.bedGraph
done

    # load the bigWig files into SQL table name friendly tables:
    mkdir /gbdb/hg19/decode
for C in female female_carrier female_noncarrier \
	male male_carrier male_noncarrier \
	sex-averaged sex-averaged_carrier sex-averaged_noncarrier
do
    N=${C}
    case ${C} in
        female) N="Female" ;;
        female_carrier) N="FemaleCarrier" ;;
        female_noncarrier) N="FemaleNonCarrier" ;;
        male) N="Male" ;;
        male_carrier) N="MaleCarrier" ;;
        male_noncarrier) N="MaleNonCarrier" ;;
        sex-averaged) N="SexAveraged" ;;
        sex-averaged_carrier) N="SexAveragedCarrier" ;;
        sex-averaged_noncarrier) N="SexAveragedNonCarrier" ;;
    esac
    echo $C $N
    rm -f /gbdb/hg19/decode/${C}.bw /gbdb/hg19/decode/${N}.bw
    ln -s `pwd`/${C}.bw /gbdb/hg19/decode/${N}.bw
    hgsql -e "drop table decode${N};" hg19
    hgBbiDbLink hg19 decode${N} /gbdb/hg19/decode/${N}.bw
done

    # compute male - female difference
    awk '{printf "%s_%d_%d\t%s\n", $1, $2, $3, $4}' male.hg19.bedGraph \
	| sort > ordered.male.txt
    awk '{printf "%s_%d_%d\t%s\n", $1, $2, $3, $4}' female.hg19.bedGraph \
	| sort > ordered.female.txt
    join ordered.male.txt ordered.female.txt > maleFemale.txt

    awk '{printf "%s\t%.6f\n", $1, $2-$3}' maleFemale.txt \
	| sed -e "s/_/\t/g" | sort -k1,1 -k2,2n > maleFemale.bedGraph
    # same result as what was lifted:
    sum maleFemale.hg19.bedGraph maleFemale.bedGraph
    #	14015  7950 maleFemale.hg19.bedGraph
    #	14015  7950 maleFemale.bedGraph


    # and hot spots
    awk '$4 > 9.99' female.hg19.bedGraph > hotSpotFemale.bed
    awk '$4 > 9.99' male.hg19.bedGraph > hotSpotMale.bed

    hgLoadBed hg19 decodeHotSpotFemale hotSpotFemale.bed
    #	Loaded 4135 elements of size 4
    hgLoadBed hg19 decodeHotSpotMale hotSpotMale.bed
    #	Loaded 4771 elements of size 4

    bedGraphToBigWig maleFemale.bedGraph /hive/data/genomes/hg19/chrom.sizes \
        MaleFemaleDifference.bw
    ln -s `pwd`/MaleFemaleDifference.bw /gbdb/hg19/decode/
    hgsql -e "drop table decodeMaleFemaleDifference;" hg19
    hgBbiDbLink hg19 decodeMaleFemaleDifference /gbdb/hg19/decode/MaleFemaleDifference.bw


#############################################################################
# DBSNP B134 / SNP134 (DONE 9/1/11)
# Redmine #5133
# Originally run 8/30/11; re-run 9/1/11 to incorporate 1000 Genomes frequency data
# that dbSNP had moved out to a new database table, SNPAlleleFreq_TGP.
    mkdir -p /hive/data/outside/dbSNP/134/human
    cd /hive/data/outside/dbSNP/134/human
    # Look at the directory listing of ftp://ftp.ncbi.nih.gov/snp/database/organism_data/
    # to find the subdir name to use as orgDir below (human_9606 in this case).
    # Then click into that directory and look for file names like
    #    b(1[0-9][0-9])_*_([0-9]+_[0-9])
    # -- use the first num for build and the second num_num for buildAssembly.
    # jkStuff/liftContigs.lft maps NCBI contig names to chroms; use that for liftUp.
    #
    # Some trial and error was required to get the config.ra just right -- assembly
    # label now has ".p2" at end, and GRCh37 patch contigs needed to be filtered out:
    cat > config.ra <<EOF
db hg19
orgDir human_9606
build 134
buildAssembly 37_2
liftUp /hive/data/genomes/hg19/jkStuff/liftContigs.lft
refAssemblyLabel GRCh37.p2
ignoreDbSnpContigs NW_0033159[0-9][0-9]
EOF
    # << emacs
    ~/kent/src/hg/utils/automation/doDbSnp.pl config.ra >& do.log & tail -f do.log


#############################################################################
# FILTER SNP134 (DONE 9/2/11 angie)
    # Redmine #5133
    # Make several tracks that are filtered subsets of snp134:
    # First, filter out the multiply-aligned and/or weight >1 SNPs -> snp134Mult
    # Second, siphon off the common variants -> snp134Common
    # Third, take the (uniquely mapped, not known to be common) variants
    # w/dbSNP's "clinically-assoc" flag -> snp134Flagged
    cd /hive/data/outside/dbSNP/134/human
    zcat snp134.bed.gz \
    | perl -we \
      '$minTotal2N = 10; \
       ($multCount, $comCount, $flagCount, $miscCount) = (0,0,0,0); \
       open($mult, "| gzip -c > snp134Mult.bed.gz") || die; \
       open($common,    "| gzip -c > snp134Common.bed.gz") || die; \
       open($flagged,   "| gzip -c > snp134Flagged.bed.gz") || die; \
       open($misc,      "| gzip -c > snp134Misc.bed.gz") || die; \
       while (<>) { \
         @w = split("\t"); \
         if ($w[16] > 1 || $w[17] =~ /MultipleAlignments/) { \
           print $mult $_; \
           $multCount++; \
         } else { \
           my ($alleleFreqCount, $nStr, $freqStr) = ($w[20], $w[22], $w[23]); \
           my @alNs = split(",", $nStr);      die unless scalar(@alNs) == $alleleFreqCount; \
           my @freqs = split(",", $freqStr);  die unless scalar(@freqs) == $alleleFreqCount; \
           my ($total2N, $maxAlleleFreq) = (0, 0); \
           for (my $i = 0;  $i < $alleleFreqCount;  $i++) { \
             $total2N += $alNs[$i]; \
             $maxAlleleFreq = $freqs[$i] if ($freqs[$i] > $maxAlleleFreq); \
           } \
           if ($alleleFreqCount >= 2 && $total2N >= $minTotal2N && $maxAlleleFreq <= 0.99) { \
             print $common $_; \
             $comCount++; \
           } elsif($w[24] =~ /clinically-assoc/)  { \
             print $flagged $_; \
             $flagCount++; \
           } else { \
             print $misc $_; \
             $miscCount++; \
           } \
         } \
       } \
       close($mult);  close($common); close($flagged);  close($misc); \
       print "snp134Mult:    $multCount\nsnp134Common:  $comCount\nsnp134Flagged: $flagCount\n" . \
             "leftover:      $miscCount\n";'
#snp134Mult:    3603177
#snp134Common:  13413905
#snp134Flagged: 26496
#leftover:      26910747

    # Load tables
    foreach subset (Mult Common Flagged)
      hgLoadBed -tab -onServer -tmpDir=/data/tmp -allowStartEqualEnd -renameSqlTable \
        hg19 snp134$subset -sqlTable=snp134.sql snp134$subset.bed.gz
    end


#############################################################################
# SNP134 ORTHOLOGOUS ALLELES IN CHIMP, ORANG, MACAQUE (DONE 9/2/11 angie)
    mkdir /hive/data/genomes/hg19/bed/snp134Ortho
    cd /hive/data/genomes/hg19/bed/snp134Ortho
    # Filter snp134 to to keep only uniquely mapped biallelic SNVs (class=single, length=1);
    zcat /hive/data/outside/dbSNP/134/human/snp134.bed.gz \
    | awk '$18 ~ /^MultipleAlignments|SingleClassTriAllelic|SingleClassQuadAllelic/ {print $4;}' \
    | sort -u \
      > snp134ExcludeIds.txt
    wc -l snp134ExcludeIds.txt
#1178007 snp134ExcludeIds.txt
    zcat /hive/data/outside/dbSNP/134/human/snp134.bed.gz \
    | awk '$3-$2 == 1 && $11 == "single" {print;}' \
    | grep -vFwf snp134ExcludeIds.txt \
      > snp134Simple.bed
    wc -l snp134Simple.bed
#32818637 snp134Simple.bed

    # Glom all human info that we need for the final table onto the
    # name, to sneak it through liftOver: rsId|chr|start|end|obs|ref|strand
    awk 'BEGIN{OFS="\t";} \
        {print $1, $2, $3, \
               $4 "|" $1 "|" $2 "|" $3 "|" $9 "|" $8 "|" $6, \
               0, $6;}' \
      snp134Simple.bed > snp134ForLiftOver.bed

    # Map coords to chimp using liftOver.
    mkdir run.liftOChimp
    cd run.liftOChimp
    mkdir split out
    splitFile ../snp134ForLiftOver.bed 10000 split/chunk
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToPanTro3.over.chain.gz \
        \{check out exists out/panTro3.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    ssh swarm
    cd /hive/data/genomes/hg19/bed/snp134Ortho/run.liftOChimp
    para make jobList
#Completed: 3282 of 3282 jobs
#CPU time in finished jobs:     314951s    5249.18m    87.49h    3.65d  0.010 y
#IO & Wait Time:                 32669s     544.49m     9.07h    0.38d  0.001 y
#Average job time:                 106s       1.77m     0.03h    0.00d
#Longest finished job:             268s       4.47m     0.07h    0.00d
#Submission to last job:           444s       7.40m     0.12h    0.01d

    # Map coords to orangutan using liftOver.
    mkdir ../run.liftOPon
    cd ../run.liftOPon
    mkdir out
    ln -s ../run.liftOChimp/split .
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToPonAbe2.over.chain.gz \
        \{check out exists out/ponAbe2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    para make jobList
#Completed: 3282 of 3282 jobs
#CPU time in finished jobs:     681601s   11360.02m   189.33h    7.89d  0.022 y
#IO & Wait Time:                 57733s     962.21m    16.04h    0.67d  0.002 y
#Average job time:                 225s       3.75m     0.06h    0.00d
#Longest finished job:             586s       9.77m     0.16h    0.01d
#Submission to last job:          1598s      26.63m     0.44h    0.02d

    # Map coords to macaque using liftOver.
    mkdir ../run.liftOMac
    cd ../run.liftOMac
    mkdir out
    ln -s ../run.liftOChimp/split .
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToRheMac2.over.chain.gz \
        \{check out exists out/rheMac2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    para make jobList
#Completed: 3282 of 3282 jobs
#CPU time in finished jobs:     826108s   13768.47m   229.47h    9.56d  0.026 y
#IO & Wait Time:                 68165s    1136.08m    18.93h    0.79d  0.002 y
#Average job time:                 272s       4.54m     0.08h    0.00d
#Longest finished job:             679s      11.32m     0.19h    0.01d
#Submission to last job:          1775s      29.58m     0.49h    0.02d

    cd /hive/data/genomes/hg19/bed/snp134Ortho
    # Concatenate the chimp results, sorting by chimp pos in order to
    # efficiently access 2bit sequence in getOrthoSeq.  The output of
    # that is then sorted by the glommed human info field, so that we
    # can use join to combine chimp and macaque results in the next step.
    # Ditto for macaque and orangutan.  Each command pipe takes ~6 minutes:
    sort -k1,1 -k2n,2n run.liftOChimp/out/panTro3.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/panTro3/panTro3.2bit \
    | sort > panTro3.orthoGlom.txt
    sort -k1,1 -k2n,2n run.liftOPon/out/ponAbe2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/ponAbe2/ponAbe2.2bit \
    | sort > ponAbe2.orthoGlom.txt
    sort -k1,1 -k2n,2n run.liftOMac/out/rheMac2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/rheMac2/rheMac2.2bit \
    | sort > rheMac2.orthoGlom.txt
    wc -l panTro3.orthoGlom.txt ponAbe2.orthoGlom.txt rheMac2.orthoGlom.txt
#30880910 panTro3.orthoGlom.txt
#29376791 ponAbe2.orthoGlom.txt
#26505681 rheMac2.orthoGlom.txt

    # Use the glommed name field as a key to join up chimp and macaque
    # allele data.  Include glommed name from both files because if only
    # file 2 has a line for the key in 2.1, then 1.1 is empty.  Then plop
    # in the orthoGlom fields from each file, which are in the same order
    # as the chimp and macaque columns of snp134OrthoPanTro2RheMac2.
    join -o '1.1 2.1 1.2 1.3 1.4 1.5 1.6 2.2 2.3 2.4 2.5 2.6' \
      -a 1 -a 2 -e '?' \
      panTro3.orthoGlom.txt ponAbe2.orthoGlom.txt \
    | awk '{if ($1 != "?") { print $1, $3,$4,$5,$6,$7,$8,$9,$10,$11,$12; } \
            else           { print $2, $3,$4,$5,$6,$7,$8,$9,$10,$11,$12; }}' \
      > tmp.txt
    join -o '1.1 2.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 2.2 2.3 2.4 2.5 2.6' \
      -a 1 -a 2 -e '?' \
      tmp.txt rheMac2.orthoGlom.txt \
    | perl -wpe 'chomp; \
        ($glom12, $glom3, $o1Chr, $o1Start, $o1End, $o1Al, $o1Strand, \
         $o2Chr, $o2Start, $o2End, $o2Al, $o2Strand, \
         $o3Chr, $o3Start, $o3End, $o3Al, $o3Strand) = split; \
        $glomKey = ($glom12 ne "?") ? $glom12 : $glom3; \
        ($rsId, $hChr, $hStart, $hEnd, $hObs, $hAl, $hStrand) = \
          split(/\|/, $glomKey); \
        $o1Start =~ s/^\?$/0/;  $o2Start =~ s/^\?$/0/;  $o3Start =~ s/^\?$/0/; \
        $o1End   =~ s/^\?$/0/;  $o2End   =~ s/^\?$/0/;  $o3End   =~ s/^\?$/0/; \
        print join("\t", $hChr, $hStart, $hEnd, $rsId, $hObs, $hAl, $hStrand, \
                         $o1Chr, $o1Start, $o1End, $o1Al, $o1Strand, \
                         $o2Chr, $o2Start, $o2End, $o2Al, $o2Strand, \
                         $o3Chr, $o3Start, $o3End, $o3Al, $o3Strand) . "\n"; \
        s/^.*$//;' \
    | sort -k1,1 -k2n,2n > snp134OrthoPt3Pa2Rm2.bed
    hgLoadBed -tab -onServer -tmpDir=/data/tmp -renameSqlTable \
      -sqlTable=$HOME/kent/src/hg/lib/snpOrthoPanPonRhe.sql \
      hg19 snp134OrthoPt3Pa2Rm2 snp134OrthoPt3Pa2Rm2.bed
#Loaded 31924973 elements of size 22
    # Cleanup:
    rm -r run*/split tmp.txt *.orthoGlom.txt bed.tab
    gzip snp134Simple.bed snp134ExcludeIds.txt snp134ForLiftOver.bed &


############################################################################
# DBSNP CODING ANNOTATIONS (134) (DONE 8/30/11 angie)
# It wasn't necessary to redo this following the 9/1 re-run of doDbSnp.pl because
# that simply picked up new allele frequency info, no change to exceptions etc.
    cd /hive/data/outside/dbSNP/134/human
    # ncbiFuncAnnotations.txt has NCBI coords: 0-based, fully closed.
    # For anything except an insertion (0 bases between flanks),
    # we need to add 1 to the end coord.  For an insertion, we need
    # to add 1 to the start coord.  Make a hash of the insertion IDs,
    # then look up each ID in ncbiFuncAnnotations.txt to tell which
    # transform to apply.
    # Note: sort -u with the keys below is too restrictive -- we need full line uniq.
    zcat ncbiFuncAnnotations.txt.gz \
    | perl -we 'open($IDS, "zcat ncbiFuncInsertions.ctg.bed.gz |") || die "ids: $!"; \
              while (<$IDS>) { chomp; $ids{$_} = 1; } \
              close($IDS); \
              %coding = (2=>1, 3=>1, 4=>1, 8=>1, 9=>1, 41=>1, 42=>1, 44=>1, 45=>1); \
              while (<>) { \
                chomp;  @w = split("\t"); # id, ctg, start, end, ... \
                next unless $coding{$w[5]}; \
                $bed4 = join("\t", $w[1], $w[2], $w[3], $w[0]); \
                if (exists $ids{$bed4} && $w[3] == $w[2]+1) { \
                  $w[2]++; # 2-base insertions: increment start coord \
                } else { \
                  $w[3]++; # increment end coord to get half-open \
                } \
                print join("\t", @w) . "\n"; \
              }' \
    | sort -k1n,1n -k2,2 -k3n,3n -k5,5 -k6n,6n \
    | uniq \
      > ncbiCodingAnnotations.txt
    wc -l ncbiCodingAnnotations.txt
#2510704 ncbiCodingAnnotations.txt
    # How many & what kinds of function types?
    cut -f 6 ncbiCodingAnnotations.txt \
    | sort -n | uniq -c
# 461567 3   (coding-synon)
#1244159 8   (cds-reference -- ignored)
#  21296 41  (nonsense)
# 729942 42  (missense)
#  52778 44  (frameshift)
#    962 45  (cds-indel)
    # Gather up multiple annotation lines into one line per {snp, gene, frame}:
    perl -e  'while (<>) { chomp; \
                my ($rsId, $ctg, $s, $e, $txId, $fxn, $frm, $nt, $aa, $codon) = split("\t"); \
                if (defined $lastRs && \
                    ($lastRs != $rsId || $lastCtg ne $ctg || $lastS != $s || \
                     $lastTx ne $txId || $lastFrm ne $frm)) { \
                  if (defined $refRow) { \
                    $fxns = "$refRow->[0],$fxns";  $nts = "$refRow->[1],$nts"; \
                    $aas = "$refRow->[2],$aas";    $codons = "$refRow->[3],$codons"; \
                  } \
                  print "$lastCtg\t$lastS\t$lastE\trs$lastRs\t$lastTx\t$lastFrm\t" . \
                        "$count\t$fxns\t$nts\t$codons\t$aas\n"; \
                  $refRow = undef;  @rows = ();  ($count, $fxns, $nts, $codons, $aas) = (); \
                } \
                ($lastRs, $lastCtg, $lastS, $lastE, $lastTx, $lastFrm) = \
                    ($rsId, $ctg, $s, $e, $txId, $frm); \
                $count++; \
                if ($fxn == 8) { \
                  $refRow = [$fxn, $nt, $aa, $codon]; \
                } else { \
                 $fxns .= "$fxn,";  $nts .= "$nt,";  $aas .= "$aa,";  $codons .= "$codon,"; \
                } \
              } \
              if (defined $refRow) { \
                $fxns = "$refRow->[0],$fxns";  $nts = "$refRow->[1],$nts"; \
                $aas = "$refRow->[2],$aas";    $codons = "$refRow->[3],$codons"; \
              } \
              print "$lastCtg\t$lastS\t$lastE\trs$lastRs\t$lastTx\t$lastFrm\t" . \
                    "$count\t$fxns\t$nts\t$codons\t$aas\n";' \
      ncbiCodingAnnotations.txt \
    | liftUp snp134CodingDbSnp.bed /hive/data/genomes/hg19/jkStuff/liftContigs.lft warn stdin
    hgLoadBed hg19 snp134CodingDbSnp -sqlTable=$HOME/kent/src/hg/lib/snp125Coding.sql \
      -renameSqlTable -tab -notItemRgb -allowStartEqualEnd \
      snp134CodingDbSnp.bed
#Loaded 1244179 elements of size 11


############################################################################
# SNPMASKED SEQUENCE FOR SNP134 (DONE 8/30/11 angie)
# It wasn't necessary to redo this following the 9/1 re-run of doDbSnp.pl because
# that simply picked up new allele frequency info, no change to exceptions etc.
    mkdir /hive/data/genomes/hg19/snp134Mask
    cd /hive/data/genomes/hg19/snp134Mask
    # Identify rsIds with various problems -- we will exclude those.
    zcat /hive/data/outside/dbSNP/134/human/snp134.bed.gz \
    | awk '$18 ~ /MultipleAlignments|ObservedTooLong|ObservedWrongFormat|ObservedMismatch|MixedObserved/ {print $4;}' \
      | sort -u \
      > snp134ExcludeRsIds.txt
    zcat /hive/data/outside/dbSNP/134/human/snp134.bed.gz \
    | grep -vFwf snp134ExcludeRsIds.txt \
      > snp134Cleaned.bed
    wc -l snp134Cleaned.bed
#37853186 snp134Cleaned.bed

    # Substitutions:
    mkdir substitutions
    snpMaskSingle snp134Cleaned.bed /hive/data/genomes/hg19/hg19.2bit stdout diffObserved.txt \
    | faSplit byname stdin substitutions/
#Masked 32668329 snps in 32666100 out of 3131050506 genomic bases
#/hive/data/genomes/hg19/hg19.2bit has 3137161264 total bases, but the total number of bases in sequences for which we masked snps is 3131050506 (difference is 6110758)
    # warnings about differing observed strings at same base position:
    wc -l diffObserved.txt
#2545 diffObserved.txt
    # Check that 6110758 is the total #bases in sequences with nothing in snp134Cleaned:
    grep -Fw single snp134Cleaned.bed | cut -f 1 | uniq > /data/tmp/1
    grep -vwf /data/tmp/1 ../chrom.sizes \
    | awk 'BEGIN {TOTAL = 0;}  {TOTAL += $2;}  END {printf "%d\n", TOTAL;}'
#6110758
    # Make sure that sizes are identical, first diffs are normal -> IUPAC,
    # and first diffs' case is preserved:
    foreach f (substitutions/chr*.fa)
      faCmp -softMask $f ../[1-9UMXY]*/$f:t |& grep -v "that differ"
    end
#chr1 in substitutions/chr1.fa differs from chr1 at ../1/chr1.fa at base 10233 (y != c)
#chr10 in substitutions/chr10.fa differs from chr10 at ../10/chr10.fa at base 60522 (K != T)
#...
#(output OK -- ambiguous bases replacing [agct] at SNP positions)
    foreach f (substitutions/chr*.fa)
      echo $f:t:r
      mv $f $f:r.subst.fa
      gzip $f:r.subst.fa &
    end

    # Insertions & deletions not done.  To date we have only offered substs for download.
    # If there is user demand, use template from snp131 above.

    # Clean up and prepare for download:
    gzip snp134Cleaned.bed &
    foreach d (substitutions)
      pushd $d
        md5sum *.gz > md5sum.txt
        cp /hive/data/genomes/hg19/snp132Mask/$d/README.txt .
      popd
    end
    # Edit the README.txt.

    # Create download links on hgwdev.
    mkdir /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/snp134Mask
    ln -s /hive/data/genomes/hg19/snp134Mask/substitutions/* \
      /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/snp134Mask/


#############################################################################
# LASTZ X. tropicalis XenTro3 (DONE - 2011-09-20 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzXenTro3.2011-09-20
    cd /hive/data/genomes/hg19/bed/lastzXenTro3.2011-09-20

    cat << '_EOF_' > DEF
# human vs X. tropicalis
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_Q=/scratch/data/blastz/HoxD55.q

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Frog xenTro3
SEQ2_DIR=/scratch/data/xenTro3/xenTro3.2bit
SEQ2_LEN=/scratch/data/xenTro3/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LAP=0
SEQ2_LIMIT=100

BASE=/hive/data/genomes/hg19/bed/lastzXenTro3.2011-09-20
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    395m51.626s
    cat fb.hg19.chainXenTro3Link.txt  
    #	87928753 bases of 2897316137 (3.035%) in intersection
    cd /hive/data/genomes/hg19/bed
    ln -s lastzXenTro3.2011-09-20 lastz.xenTro3

    #	running the swap - DONE - 2011-09-21
    mkdir /hive/data/genomes/xenTro3/bed/blastz.hg19.swap
    cd /hive/data/genomes/xenTro3/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzXenTro3.2011-09-20/DEF \
	-chainMinScore=5000 -chainLinearGap=loose \
	-workhorse=hgwdev -smallClusterHub=memk -bigClusterHub=swarm \
	-swap > swap.log 2>&1 &
    #	real    37m7.001s
    cat fb.xenTro3.chainHg19Link.txt 
    #	90929066 bases of 1358334882 (6.694%) in intersection

############################################################################
# GENEREVIEWS TRACK (DONE 2011-09-22 - Chin)
    mkdir /hive/data/genomes/hg19/bed/geneReviews
    mkdir -p /hive/data/outside/ncbi/geneReviews/current
    cd /hive/data/outside/ncbi/geneReviews/current
    wget --timestamping ftp://ftp.ncbi.nih.gov/pub/GeneTests/README.html
    wget --timestamping ftp://ftp.ncbi.nih.gov/pub/GeneTests/*.txt
    # Note: the report *.txt are updated daily
    wc -l *.txt
    219 README.html
    #   3043 data_to_build_custom_reports.txt
    #   7245 disease_OMIM.txt
    #   3729 disease_gene_GR.txt
    #   1415 disease_hierarchy.txt

    cp disease_gene_GR.txt /hive/data/genomes/hg19/bed/geneReviews/.
    cd /hive/data/genomes/hg19/bed/geneReviews
    cat disease_gene_GR.txt | grep -v "^#" \
     | awk  -F'|' '{if ($3!="-" && $3!="Not applicable" && $4!="-") \
      printf  "%s\t%s\t%s\t%s\n", $3, $4, $1, $2}' | sort -k1 > grRefGeneData.tab
    # Create geneReviewsRefGene table
    cat << '_EOF_' > $HOME/kent/src/hg/lib/geneReviewsRefGene.sql
CREATE TABLE geneReviewsRefGene (
    geneSymbol  varchar(255) not null,   # refSeq gene symbol         
    grShort     varchar(255) not null,   # short name for GeneReviews article
    diseaseID   int unsigned not null,   # Disease ID of the review article
    diseaseName varchar(255) not null,   # Disease name of the review article
    index (geneSymbol)
);
'_EOF_'
    # << happy emacs

    # load RefSeg Gene to geneReview mapping list to hg19
    hgLoadSqlTab -warn hg19 geneReviewsRefGene \
      $HOME/kent/src/hg/lib/geneReviewsRefGene.sql grRefGeneData.tab
    # Scanning through 1 files

    # Generate a list of refSeq genes thah have geneReview assoicate with it.
    cat  grRefGeneData.tab | awk -F'\t' '{printf  "%s\n", $1}' \
       | sort  | uniq  > grRefGene.lst
    wc -l *.*
    #   3729 disease_gene_GR.txt
    #   1111 grData.tab
    #    946 grRefGene.lst
  
    # for each refGen in grRefGene.lst, create a non-overlapping bed row.
    cat grRefGene.lst | while read G
      do
        echo ${G}
        hgsql hg19 -N -e \
          "SELECT e.chrom,e.txStart,e.txEnd,j.geneSymbol \
          FROM knownGene e, kgXref j WHERE e.alignID = j.kgID AND \
          j.geneSymbol ='${G}' ORDER BY e.chrom,e.txStart;" > temp.in
        bedRemoveOverlap temp.in temp.out
        cat temp.out >> geneReviews.tab
      done
    rm temp.*

    # load the collapsed bed4 file to hg19,
    hgLoadBed hg19 geneReviews geneReviews.tab

    # addGeneReviewToBed.pl will add the geneReviews detail in html format to 
    # the bed 4 file
    cat << '_EOF_' > $HOME/kent/src/utils/geneReviews/addGeneReviewToBed.pl
#!/usr/bin/perl
use warnings;
use strict;
sub usage() {
    print "usage: ./addGRtoBed.pl dbName > outputFile\n";
}
my $argc=scalar(@ARGV);
if ($argc != 1) {
    usage; die "ERROR: Please supply a database name for results.\n";
}
#get the list of (unique) of gene symbols form geneReviews table
my @geneReviews = split('\n',
 `hgsql -N -e "select chrom, chromStart, chromEnd, name from geneReviews;" $ARGV[0]`);
my $grBed;
my $clickMsg = " (Click links below to search GeneReviews or GeneTests)";
my $firstTime = 1;
my $field;
my $details;
foreach $grBed(@geneReviews) {
   $details = "";
   my @col = split(/\t/, $grBed);
   #print "Processing name: ", $col[3], " <BR>";
   my @grShort = split('\n',
    `hgsql -N -e 'select  grShort, diseaseID, diseaseName from geneReviewsRefGene where geneSymbol="$col[3]"' $ARGV[0]`);
   $firstTime = 1;
   my $count = scalar(@grShort);
   my $i;
   my $j;
   for ($i=0; $i < $count; $i++) {
       my @f5 = split(/\t/, $grShort[$i]);
   if ($firstTime == 1) {
      $firstTime=0;
      $details = "";
      $details = "<BR><B>GeneReview available for " . $col[3] .  ": </B> " . $clickMsg . "<BR>";
      $details .= "<PRE><TT>";
      $details .= "Short name    Disease ID     GeneTests disease name<BR>";
      $details .= "-----------------------------------------------------------";
      $details .= "-----------------------------------------------------------";
      $details .= "----------------------------------<BR>";
      } 
      $details .= "<A HREF=\"http://www.ncbi.nlm.nih.gov/books/n/gene/" . $f5[0] . "\" TARGET=_blank><B>" . $f5[0] . "</B></A>";
      if (length($f5[0]) <= 15) {
        for ($j = 0; $j <  15-length($f5[0]); $j ++ )
           {
              $details .= " ";
           }
         }
       $details .=  $f5[1] . "       ";
       $details .= "<A HREF=\"http://www.ncbi.nlm.nih.gov/sites/GeneTests/review/disease/" . $f5[2] . "?db=genetests&search_param==begins_with\" TARGET=_blank>" . $f5[2]  ."<BR>";
     }
       $details .= "</TT></PRE><BR>";
       print $col[0], "\t", $col[1], "\t", $col[2], "\t", $col[3], "\t", $details, "\n";
  }
'_EOF_'
    # << happy emacs
    chmod +x $HOME/kent/src/utils/geneReviews/addGeneReviewToBed.pl
    # Add geneReview item in html format format as field 5
    $HOME/kent/src/utils/geneReviews//addGeneReviewToBed.pl hg19 > hg19.geneReviews.bed5

    # Convert to bigBed format 
    cat << '_EOF_' > $HOME/kent/src/hg/lib/geneReviewsBed5.as
table geneReviewsBed5
"GeneReviews  bigBed 4 + with extra field for detail page"
    (
    string chrom;      "Reference sequence chromosome or scaffold"
    uint   chromStart; "Start position in chromosome"
    uint   chromEnd;   "End position in chromosome"
    string name;       "Short Name of item"
    lstring description; "geneReviews item details in html"
    )
'_EOF_'
    # << happy emacs

    /cluster/bin/x86_64/bedToBigBed -bedFields=4 -tabs \
     -as=$HOME/kent/src/hg/lib/geneReviewsBed5.as  hg19.geneReviews.bed5 \
     /hive/data/genomes/hg19/chrom.sizes hg19.geneReviews.bb

    # upload the bigBed file to genomewiki
    /cluster/bin/scripts/gwUploadFile hg19.geneReviews.bb hg19.geneReviews.bb
    #  UploadFile hg19.geneReviews.bb hg19.geneReviews.bb
    #  # loading file: hg19.geneReviews.bb
    #  # into Image name: Hg19.geneReviews.bb
    #  # login name: chinhli
    #  # siteUrl: genomewiki.ucsc.edu
    #  # traceBackLimit: 0
    #  # traceBackLimit: 0 past site.Images
    #  Image info: {u'comment': u'gwUploadFile upload', u'sha1': u'643966466503bd6770f67d49397f4b94174beed2', u'url': u'http://genomewiki.ucsc.edu/images/b/b9/Hg19.geneReviews.bb', u'timestamp': u'2011-09-22T17:40:56Z', u'metadata': None, u'height': 0, u'width': 0, u'user': u'Chinhli', u'descriptionurl': u'http://genomewiki.ucsc.edu/index.php/File:Hg19.geneReviews.bb', u'size': 174667}
    #  Image File:Hg19.geneReviews.bb usage:

    cat << '_EOF_' > $HOME/kent/src/hg/lib/geneReviewsBB.sql
# sql to create geneReviewsBB table 
DROP TABLE IF EXISTS geneReviewsBB;
CREATE TABLE geneReviewsBB (
    fileName varchar(255) not null      # geneReviews.bb location (url)
);
'_EOF_' 
    # << happy emacs
    hgsql hg19 -e "source $HOME/kent/src/hg/lib/geneReviewsBB.sql;"
    hgsql hg19 -e 'insert into geneReviewsBB values ("http://genomewiki.ucsc.edu/images/b/b9/Hg19.geneReviews.bb");'


##############################################################################
# hgPal downloads redone for new knownGene (re-DONE 2011-10-18 braney)
#   FASTA from 46way for  knownGene, knownCanonical 

    ssh hgwdev
    screen
    bash
#    rm -rf /cluster/data/hg19/bed/multiz46way/pal
#    mkdir /cluster/data/hg19/bed/multiz46way/pal
    cd /cluster/data/hg19/bed/multiz46way/pal
    for i in `cat ../species.list`; do echo $i; done > order.lst

    mz=multiz46way
    gp=knownGene
    db=hg19
    mkdir exonAA exonNuc ppredAA ppredNuc
    for j in `sort -nk 2 /cluster/data/$db/chrom.sizes | awk '{print $1}'`
    do
	echo "date"
	echo "mafGene -chrom=$j  $db $mz $gp order.lst stdout | \
	    gzip -c > ppredAA/$j.ppredAA.fa.gz"
	echo "mafGene -chrom=$j -noTrans $db $mz $gp order.lst stdout | \
	    gzip -c > ppredNuc/$j.ppredNuc.fa.gz"
	echo "mafGene -chrom=$j -exons -noTrans $db $mz $gp order.lst stdout | \
	    gzip -c > exonNuc/$j.exonNuc.fa.gz"
	echo "mafGene -chrom=$j -exons $db $mz $gp order.lst stdout | \
	    gzip -c > exonAA/$j.exonAA.fa.gz"
    done > $gp.$mz.jobs

    time sh -x $gp.$mz.jobs > $gp.$mz.job.log 2>&1 &
    sleep 1
    tail -f $gp.$mz.job.log

# real    258m41.548s
# user    37m23.511s
# sys     6m47.978s

    mz=multiz46way
    gp=knownGene
    db=hg19

    zcat exonAA/c*.gz | gzip -c > $gp.$mz.exonAA.fa.gz
    zcat exonNuc/c*.gz | gzip -c > $gp.$mz.exonNuc.fa.gz
    zcat ppredAA/c*.gz | gzip -c > $gp.$mz.ppredAA.fa.gz
    zcat ppredNuc/c*.gz | gzip -c > $gp.$mz.ppredNuc.fa.gz

    rm -rf exonAA exonNuc ppredAA ppredNuc

    mz=multiz46way
    gp=knownGene
    db=hg19
    pd=/usr/local/apache/htdocs-hgdownload/goldenPath/$db/$mz/alignments
    mkdir -p $pd
    ln -s `pwd`/$gp.$mz.exonAA.fa.gz $pd/$gp.exonAA.fa.gz
    ln -s `pwd`/$gp.$mz.exonNuc.fa.gz $pd/$gp.exonNuc.fa.gz

    # now do the canonical set
    cd /cluster/data/hg19/bed/multiz46way/pal
    mz=multiz46way
    gp=knownCanonical
    db=hg19
    for j in `awk '{print $1}' /cluster/data/hg19/chrom.sizes`
    do
	echo "select chrom, chromStart, chromEnd, transcript from knownCanonical where chrom='$j'" | hgsql $db | tail -n +2 > $j.known.bed
    done

    mkdir exonAA exonNuc ppredAA ppredNuc
    for j in `sort -nk 2 /cluster/data/$db/chrom.sizes | awk '{print $1}'`
    do
	echo "date"
	echo "mafGene -geneBeds=$j.known.bed  $db $mz knownGene order.lst stdout | \
	    gzip -c > ppredAA/$j.ppredAA.fa.gz"
	echo "mafGene -geneBeds=$j.known.bed -noTrans $db $mz knownGene order.lst stdout | \
	    gzip -c > ppredNuc/$j.ppredNuc.fa.gz"
	echo "mafGene -geneBeds=$j.known.bed -exons -noTrans $db $mz knownGene order.lst stdout | \
	    gzip -c > exonNuc/$j.exonNuc.fa.gz"
	echo "mafGene -geneBeds=$j.known.bed -exons $db $mz knownGene order.lst stdout | \
	    gzip -c > exonAA/$j.exonAA.fa.gz"
    done > $gp.$mz.jobs

    time sh -x $gp.$mz.jobs > $gp.$mz.job.log 2>&1 &
    sleep 1
    tail -f $gp.$mz.job.log

# real    255m47.897s
# user    14m38.822s
# sys     5m32.823s

    rm *.known.bed
    mz=multiz46way
    gp=knownCanonical
    db=hg19
    zcat exonAA/c*.gz | gzip -c > $gp.$mz.exonAA.fa.gz
    zcat exonNuc/c*.gz | gzip -c > $gp.$mz.exonNuc.fa.gz
    zcat ppredAA/c*.gz | gzip -c > $gp.$mz.ppredAA.fa.gz
    zcat ppredNuc/c*.gz | gzip -c > $gp.$mz.ppredNuc.fa.gz

    rm -rf exonAA exonNuc ppredAA ppredNuc

    mz=multiz46way
    gp=knownCanonical
    db=hg19
    pd=/usr/local/apache/htdocs-hgdownload/goldenPath/$db/$mz/alignments
    mkdir -p $pd
    ln -s `pwd`/$gp.$mz.exonAA.fa.gz $pd/$gp.exonAA.fa.gz
    ln -s `pwd`/$gp.$mz.exonNuc.fa.gz $pd/$gp.exonNuc.fa.gz
############################################################################
# UPDATE COSMIC TRACK (DONE 2011-10-11 Fan)

mkdir /hive/data/outside/cosmic/20111011
# put raw data file, UCSCMutExp_v55_090911.csv, received by email to there.

mkdir /hive/data/genomes/hg19/bed/cosmic/20111011
cd /hive/data/genomes/hg19/bed/cosmic/20111011

cp -p /hive/data/outside/cosmic/20111011/UCSCMutExp_v55_090911.csv .

cat UCSCMutExp_v55_090911.csv|sed -e 's/\t//g' |sed -e 's/,/\t/g' |\
grep -v COSMIC_MUTATION_ID |grep -v 'selected'|grep COSM >UCSCMutExp_v55_090911.tab

hgsql hg19 -e 'drop table cosmicRaw'
hgsql hg19 < ~/kent/src/hg/lib/cosmicRaw.sql

hgLoadSqlTab hg19 cosmicRaw ~/kent/src/hg/lib/cosmicRaw.sql UCSCMutExp_v55_090911.tab

# use  grch37_start-1 for our zero based chromStart and 
# conver their chr23 and chr24 to chrX and chrY.

hgsql hg19 -N -e 'select "chr", chromosome, grch37_start-1, grch37_stop, cosmic_mutation_id from cosmicRaw' \
|grep -v NULL |sed -e 's/chr\t/chr/'|sort -u|sed -e 's/chr23/chrX/' |sed -e 's/chr24/chrY/' >cosmic.bed

hgLoadBed -allowStartEqualEnd  hg19 cosmic cosmic.bed

#############################################################################
# LASTZ Gorilla GorGor3 (DONE - 2011-10-17 - Hiram)
    mkdir /hive/data/genomes/hg19/bed/lastzGorGor3.2011-10-17
    cd /hive/data/genomes/hg19/bed/lastzGorGor3.2011-10-17

    cat << '_EOF_' > DEF
# human vs gorilla
# maximum M allowed with lastz is only 254
BLASTZ_M=254
BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q
BLASTZ_O=600
BLASTZ_E=150
# other parameters on advice from Webb
BLASTZ_K=4500
BLASTZ_Y=15000
BLASTZ_T=2

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Frog gorGor3
SEQ2_DIR=/scratch/data/gorGor3/gorGor3.2bit
SEQ2_LEN=/scratch/data/gorGor3/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LAP=0
SEQ2_LIMIT=100

BASE=/hive/data/genomes/hg19/bed/lastzGorGor3.2011-10-17
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #	establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	`pwd`/DEF \
	-syntenicNet \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> do.log 2>&1 &
    #	real    159m46.598s
    cat fb.hg19.chainGorGor3Link.txt  
    #	2603997992 bases of 2897316137 (89.876%) in intersection

    cd /hive/data/genomes/hg19/bed
    ln -s lastzGorGor3.2011-10-17 lastz.gorGor3

    #	running the swap - DONE - 2011-09-21
    mkdir /hive/data/genomes/gorGor3/bed/blastz.hg19.swap
    cd /hive/data/genomes/gorGor3/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
	/hive/data/genomes/hg19/bed/lastzGorGor3.2011-10-17/DEF \
	-swap -syntenicNet \
	-noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
	-workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
	> swap.log 2>&1 &
    #	real    69m39.685s
    cat fb.gorGor3.chainHg19Link.txt 
    #	2571797450 bases of 2822760080 (91.109%) in intersection

############################################################################
# ISCA FROM DBVAR (DONE 10/18/11 angie)
    # Redmine: Track #34 (dbVar for human)
    set today = `date +%Y_%m_%d`
    mkdir /hive/data/genomes/hg19/bed/isca/$today
    cd /hive/data/genomes/hg19/bed/isca/$today
    # Get variants submitted on this assembly, and variants remapped from other assemblies.
    wget ftp://ftp.ncbi.nlm.nih.gov/pub/dbVar/data/Homo_sapiens/by_study/nstd37_ISCA_2011/gvf/nstd37_ISCA_2011.GRCh37.submitted.all.germline.ucsc.gvf.gz
    wget ftp://ftp.ncbi.nlm.nih.gov/pub/dbVar/data/Homo_sapiens/by_study/nstd37_ISCA_2011/gvf/nstd37_ISCA_2011.GRCh37.remap.all.germline.ucsc.gvf.gz
    zcat nstd37_ISCA*.gvf.gz \
    | perl -wne 'next if (/^(#|browser|track)/); chomp; \
        ($c, undef, $varType, $s, $e, $score, $strand, undef, $attrs) = split("\t"); \
        ($thickS, $thickE) = ($s, $e); \
        ($attrCount, $attrTags, $attrVals, $gotVarTypeAttr) = (0, "", "", 0); \
        foreach my $tagVal (split(";", $attrs)) { \
          ($tag, $val) = split("=", $tagVal); \
          if ($tag eq "ID") {$name = $val;} \
          elsif ($tag eq "Start_range") { \
            ($outerStart, $innerStart) = split(",", $val); \
            $s = $outerStart if ($outerStart =~ /^\d+$/ && $outerStart < $s); \
            $thickS = $innerStart if ($innerStart =~ /^\d+$/ && $innerStart > $thickS); \
          } elsif ($tag eq "End_range") { \
            ($innerEnd, $outerEnd) = split(",", $val); \
            $thickE = $innerEnd if ($innerEnd =~ /^\d+$/ && $innerEnd < $thickE); \
            $e = $outerEnd if ($outerEnd =~ /^\d+$/ && $outerEnd > $e); \
          } elsif ($tag eq "var_type") { \
            $gotVarTypeAttr = 1; \
          } \
          # Escape commas so the val list parses correctly in autoSql: \
          $val =~ s/%/%25/g;   $val =~ s/,/%2C/g; \
          $attrTags .= "$tag,";  $attrVals .= "$val,";  $attrCount++; \
        } \
        next if ($name =~ /^[ne]sv\d/); # keep only supporting variants (ssv) \
        if (! $gotVarTypeAttr) { \
          $varType =~ s/%/%25/g;   $varType =~ s/,/%2C/g; \
          $attrTags .= "var_type,";  $attrVals .= "$varType,";  $attrCount++; \
        } \
        @bed8Attrs = ($c, $s-1, $e, $name, $score, $strand, $thickS-1, $thickE, \
                      $attrCount, $attrTags, $attrVals); \
        print join("\t", @bed8Attrs) . "\n";' \
      > isca.bed
    wc -l isca.bed
#11542 isca.bed
    # Split into subtracks by clinical_int value.
    zcat nstd37_ISCA*.gvf.gz \
    | grep ssv | sed -e 's/.*clinical_int=//; s/;.*//;' | sort | uniq -c
#   4250 Benign
#   4063 Pathogenic
#   2835 Uncertain Significance
#    333 Uncertain Significance: likely benign
#     61 Uncertain Significance: likely pathogenic
    foreach subtrack (Benign Pathogenic)
      grep -w  $subtrack isca.bed > isca$subtrack.bed
      hgLoadBed -tab -renameSqlTable -sqlTable=$HOME/kent/src/hg/lib/bed8Attrs.sql \
        -allowStartEqualEnd hg19 isca$subtrack isca$subtrack.bed
    end
#Loaded 4250 elements of size 11
#Loaded 4063 elements of size 11

    # The subcategories of Uncertain need a bit more sophisticated treatment:
    set subtrack = Uncertain
    grep -w $subtrack isca.bed \
    | grep -v 'Uncertain Significance: likely' \
      > isca$subtrack.bed
    hgLoadBed -tab -renameSqlTable -sqlTable=$HOME/kent/src/hg/lib/bed8Attrs.sql \
      -allowStartEqualEnd hg19 isca$subtrack isca$subtrack.bed
#Loaded 2835 elements of size 11
    foreach unc (benign pathogenic)
      set subtrack = Likely`perl -we 'print ucfirst("'$unc'");'`
      grep -w "Uncertain Significance: likely $unc" isca.bed \
        > isca$subtrack.bed
      hgLoadBed -tab -renameSqlTable -sqlTable=$HOME/kent/src/hg/lib/bed8Attrs.sql \
        -allowStartEqualEnd hg19 isca$subtrack isca$subtrack.bed
    end
#Loaded 333 elements of size 11
#Loaded 61 elements of size 11


############################################################################
# LINCRNAS FROM ISCA FROM BROAD (DONE 2011-10-10 Chin)
# Human lincRNA Catalog

    # unzip data from Board to /hive/data/outside/lincRnaFromCabili
    mkdir /hive/data/genomes/hg19/bed/lincRnaFromCabili
    cd /hive/data/genomes/hg19/bed/lincRnaFromCabili
    cp /hive/data/outside/lincRnaFromCabili/Cabili_etal_BodyMaplincRNAs.key.txt .
    cp /hive/data/outside/lincRnaFromCabili/Cabili_etal_description.doc .
    cp /hive/data/outside/lincRnaFromCabili/Cabili_etal_BodyMapLincRNAs.bed .
    cp /hive/data/outside/lincRnaFromCabili/lincRNAs_transcripts.gtf .

    # Load data for lincRNAsTranscripts track	
    cd /hive/data/genomes/hg19/bed/lincRnaFromCabili
    ldHgGene -gtf hg19 lincRNAsTranscripts lincRNAs_transcripts.gtf
    # Read 21630 transcripts in 67096 lines in 1 files
    #   21630 groups 43 seqs 7 sources 1 feature types
    # 21630 gene predictions

    cat << '_EOF_' > createExpDataByCellType.pl
#!/usr/bin/perl
# Create expData table form a microarray bed15 file
# for further analysis
# usage: ./createExpDataByCellType.pl <bed15File> 
use strict;
use warnings;
my $line;
my @bF;
my $i;
my $outFname;
my $name;
my $expCount;
my $expIds;
my $expScores;
my @expId;
my @expScore;
my $score;
my $tScore;
my $fScore;
my $log2 = log(2);
my $sLog2;
my $tLog2;
# Define the 22 cell type array
my @cellType = ("Adipose","Adrenal","Brain","Breast","Colon","Heart","Kidney",
              "Liver","Lung","LymphNode","Ovary","Prostate","SkeletalMuscle",
              "WhiteBloodCell","Testes","Thyroid","Testes_R","Brain_R",
              "Placenta_R","Foreskin_R","hLF_r2","hLF_r1");
#Read in the microarray (bed15) files
# Assume number of exp and score agreed
my $argc=scalar(@ARGV);
if ($argc < 1)
   {
      print "usage: ./createExpDataByCellType.pl <bed15File>";
    }
my $fName = $ARGV[0];
my $outFName;
# Loop thru each cell type
for($i = 0; $i < scalar(@cellType); $i++) {
  open(FHIN, $fName) or die "Can not open $fName";
  $outFName= "lincRNAsCT" . $cellType[$i] . "\.tab";
  open(FHOUT, ">$outFName") or die "Can not open $outFName";
  while ($line = <FHIN>) {
     chomp($line);
     @bF = split('\t', $line);
     printf(FHOUT "%s\t%s\t%s\t%s\t",
                  $bF[0],$bF[1],$bF[2],$bF[3]); 
     $bF[14] =~ s/,$//;
     $expScores = $bF[14];
     @expScore = split(",",$expScores);
     # Process the expRatio
     $tLog2 = log($expScore[$i] + 0.5)/$log2;
     $sLog2 = sprintf("%.3f",$tLog2);
     # scale sLog2 using 0 (-1) .. 1000 (4)
     if ($sLog2 <= 4.0) {
         $tScore = ($sLog2 + 1) * (1000/5);
         $fScore = sprintf("%3d",$tScore);
       } else {
         $tScore = 1000;
       }
     if ($tScore >= 1000) {
         $fScore = 1000;
       } else {
         $fScore = sprintf("%3d",$tScore);
       }
     printf(FHOUT "%s\t%s\t%s\n",$fScore, $expScore[$i],$sLog2);     
  } # end while
  close FHIN;
  close FHOUT;
} # for loop
'_EOF_'
    # << happy emacs
    chmod +x createExpDataByCellType.pl
    ./createExpDataByCellType.pl Cabili_etal_BodyMapLincRNAs.bed 

    cat << '_EOF_' > deleteCTTables.sh
#!/bin/sh
#
cellType=(Adipose Adrenal Brain Breast Colon Heart Kidney
              Liver Lung LymphNode Ovary Prostate SkeletalMuscle
              WhiteBloodCell Testes Thyroid Testes_R Brain_R
              Placenta_R Foreskin_R hLF_r2 hLF_r1)
for c in "${cellType[@]}"
do
    echo Processing lincRNAs$c
    hgsql  hg19 -e "DROP TABLE IF EXISTS  lincRNAsCT$c;"
done
'_EOF_'
    # << happy emacs
    chmod +x deleteCTTables.sh
    ./deleteCTTables.sh
   
    cat << '_EOF_' > lincRNAsCTTemp.sql
CREATE TABLE lincRNAsCTTemp (
    chrom varchar(255) not null,        # Human chromosome or FPC contig
    chromStart int unsigned not null,   # Start position in chromosome
    chromEnd int unsigned not null,     # End position in chromosome
    name varchar(255) not null,         # Name of item
    score int unsigned not null,        # Score from 0-1000
    rawScore float not null,            # Raw Signal Score
    log2RawScore float not null         # log2 of raw score
);
'_EOF_'
    # << happy emacs

    cat << '_EOF_' > loadLincRNAsAllCellType.sh
#!/bin/sh
#
cellType=(Adipose Adrenal Brain Breast Colon Heart Kidney
              Liver Lung LymphNode Ovary Prostate SkeletalMuscle
              WhiteBloodCell Testes Thyroid Testes_R Brain_R
              Placenta_R Foreskin_R hLF_r2 hLF_r1)
for c in "${cellType[@]}"
do
    echo Processing lincRNAsCT$c.tab
    hgLoadBed -tab -sqlTable=lincRNAsCTTemp.sql hg19 lincRNAsCTTemp lincRNAsCT$c.tab
    hgsql hg19 -e "RENAME TABLE lincRNAsCTTemp to lincRNAsCT$c"
done
'_EOF_'
    # << happy emacs
    chmod +x loadLincRNAsAllCellType.sh
    ./loadLincRNAsAllCellType.sh

#############################################################################
# LASTZ Gibbon NomLeu1 (DONE - 2011-11-04 - Chin)

    mkdir /hive/data/genomes/hg19/bed/lastzNomLeu1.2011-11-04
    cd /hive/data/genomes/hg19/bed/lastzNomLeu1.2011-11-04

    cat << '_EOF_' > DEF
# human vs gibbon
# maximum M allowed with lastz is only 254
BLASTZ_M=254
BLASTZ_Q=/scratch/data/blastz/human_chimp.v2.q
BLASTZ_O=600
BLASTZ_E=150
# other parameters on advice from Webb
BLASTZ_K=4500
BLASTZ_Y=15000
BLASTZ_T=2

# TARGET: Human hg19
SEQ1_DIR=/scratch/data/hg19/nib
SEQ1_LEN=/scratch/data/hg19/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Gibbon nomLeu1
SEQ2_DIR=/scratch/data/nomLeu1/nomLeu1.2bit
SEQ2_LEN=/scratch/data/nomLeu1/chrom.sizes
SEQ2_CHUNK=20000000
SEQ2_LAP=0
SEQ2_LIMIT=100

BASE=/hive/data/genomes/hg19/bed/lastzNomLeu1.2011-11-04
TMPDIR=/scratch/tmp
'_EOF_'
    # << happy emacs

    #   establish a screen to control this job
    screen
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
        `pwd`/DEF \
        -syntenicNet \
        -noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
        -workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
        > do.log 2>&1 &
    #   real    724m15s
    cat fb.hg19.chainNomLeu1Link.txt
    #   2543943556 bases of 2897316137 (87.803%) in intersection

    cd /hive/data/genomes/hg19/bed
    ln -s lastzNomLeu1.2011-11-04 lastz.nomLeu1

    #   running the swap - DONE - 2011-11-08
    mkdir /hive/data/genomes/nomLeu1/bed/blastz.hg19.swap
    cd /hive/data/genomes/nomLeu1/bed/blastz.hg19.swap
    time nice -n +19 doBlastzChainNet.pl -verbose=2 \
        /hive/data/genomes/hg19/bed/lastzNomLeu1.2011-11-04/DEF \
        -swap -syntenicNet \
        -noLoadChainSplit -chainMinScore=5000 -chainLinearGap=medium \
        -workhorse=hgwdev -smallClusterHub=encodek -bigClusterHub=swarm \
        > swap.log 2>&1 &
    #   real     69m27s
    cat fb.nomLeu1.chainHg19Link.txt
    #   2480558770 bases of 2756591777 (89.986%) in intersection


#############################################################################
# DBSNP B135 / SNP135 (DONE 11/9/11)
# Redmine #5170
    mkdir -p /hive/data/outside/dbSNP/135/human
    cd /hive/data/outside/dbSNP/135/human
    # Look at the directory listing of ftp://ftp.ncbi.nih.gov/snp/database/organism_data/
    # to find the subdir name to use as orgDir below (human_9606 in this case).
    # Then click into that directory and look for file names like
    #    b(1[0-9][0-9])_*_([0-9]+_[0-9])
    # -- use the first num for build and the second num_num for buildAssembly.
    # jkStuff/liftContigs.lft maps NCBI contig names to chroms; use that for liftUp.
    #
    # Some trial and error was required to get the config.ra just right -- assembly
    # label now has ".p5" at end despite buildAssembly being 37_3, and more GRCh37
    # patch contigs needed to be filtered out:
    cat > config.ra <<EOF
db hg19
orgDir human_9606
build 135
buildAssembly 37_3
liftUp /hive/data/genomes/hg19/jkStuff/liftContigs.lft
refAssemblyLabel GRCh37.p5
ignoreDbSnpContigs NW_003(3159[0-9][0-9]|5710[3-6][0-9])
EOF
    ~/kent/src/hg/utils/automation/doDbSnp.pl config.ra >& do.log & tail -f do.log
    # Sent dbSNP some emails about conditions causing lots of SNPs in snp135Errors.bed.gz
    # due to inconsistent locType vs. coords.  2.3M SNPs have new exception SingleAlleleFreq,
    # and 474k SNPs have new exception InconsistentAlleles (allele freqs vs observed).


#############################################################################
# FILTER SNP135 (DONE 11/14/11 angie)
    # Redmine #5170
    # Make several tracks that are filtered subsets of snp135:
    # First, filter out the multiply-aligned and/or weight >1 SNPs -> snp135Mult
    # Second, siphon off the common variants -> snp135Common
    # Third, take the (uniquely mapped, not known to be common) variants
    # w/dbSNP's "clinically-assoc" flag -> snp135Flagged
    cd /hive/data/outside/dbSNP/135/human
    zcat snp135.bed.gz \
    | perl -we \
      '$minTotal2N = 10; \
       ($multCount, $comCount, $flagCount, $miscCount) = (0,0,0,0); \
       open($mult, "| gzip -c > snp135Mult.bed.gz") || die; \
       open($common,    "| gzip -c > snp135Common.bed.gz") || die; \
       open($flagged,   "| gzip -c > snp135Flagged.bed.gz") || die; \
       open($misc,      "| gzip -c > snp135Misc.bed.gz") || die; \
       while (<>) { \
         @w = split("\t"); \
         if ($w[16] > 1 || $w[17] =~ /MultipleAlignments/) { \
           print $mult $_; \
           $multCount++; \
         } else { \
           my ($alleleFreqCount, $nStr, $freqStr) = ($w[20], $w[22], $w[23]); \
           my @alNs = split(",", $nStr);      die unless scalar(@alNs) == $alleleFreqCount; \
           my @freqs = split(",", $freqStr);  die unless scalar(@freqs) == $alleleFreqCount; \
           my ($total2N, $maxAlleleFreq) = (0, 0); \
           for (my $i = 0;  $i < $alleleFreqCount;  $i++) { \
             $total2N += $alNs[$i]; \
             $maxAlleleFreq = $freqs[$i] if ($freqs[$i] > $maxAlleleFreq); \
           } \
           if ($alleleFreqCount >= 2 && $total2N >= $minTotal2N && $maxAlleleFreq <= 0.99) { \
             print $common $_; \
             $comCount++; \
           } elsif($w[24] =~ /clinically-assoc/)  { \
             print $flagged $_; \
             $flagCount++; \
           } else { \
             print $misc $_; \
             $miscCount++; \
           } \
         } \
       } \
       close($mult);  close($common); close($flagged);  close($misc); \
       print "snp135Mult:    $multCount\nsnp135Common:  $comCount\nsnp135Flagged: $flagCount\n" . \
             "leftover:      $miscCount\n";'
#snp135Mult:    3538479
#snp135Common:  11525489
#snp135Flagged: 32077
#leftover:      39116035

    # Load tables
    foreach subset (Mult Common Flagged)
      hgLoadBed -tab -onServer -tmpDir=/data/tmp -allowStartEqualEnd -renameSqlTable \
        hg19 snp135$subset -sqlTable=snp135.sql snp135$subset.bed.gz
    end


#############################################################################
# SNP135 ORTHOLOGOUS ALLELES IN CHIMP, ORANG, MACAQUE (DONE 11/14/11 angie)
    mkdir /hive/data/genomes/hg19/bed/snp135Ortho
    cd /hive/data/genomes/hg19/bed/snp135Ortho
    # Filter snp135 to to keep only uniquely mapped biallelic SNVs (class=single, length=1);
    zcat /hive/data/outside/dbSNP/135/human/snp135.bed.gz \
    | awk '$18 ~ /^MultipleAlignments|SingleClassTriAllelic|SingleClassQuadAllelic/ {print $4;}' \
    | sort -u \
      > snp135ExcludeIds.txt
    wc -l snp135ExcludeIds.txt
#1297409 snp135ExcludeIds.txt
    zcat /hive/data/outside/dbSNP/135/human/snp135.bed.gz \
    | awk '$3-$2 == 1 && $11 == "single" {print;}' \
    | grep -vFwf snp135ExcludeIds.txt \
#NOTE FOR NEXT TIME: pipe output straight to awk command below... don't need this 7G intermediate:
      > snp135Simple.bed
    wc -l snp135Simple.bed
#44228667 snp135Simple.bed

    # Glom all human info that we need for the final table onto the
    # name, to sneak it through liftOver: rsId|chr|start|end|obs|ref|strand
    awk 'BEGIN{OFS="\t";} \
        {print $1, $2, $3, \
               $4 "|" $1 "|" $2 "|" $3 "|" $9 "|" $8 "|" $6, \
               0, $6;}' \
      snp135Simple.bed > snp135ForLiftOver.bed

    # Map coords to chimp using liftOver.
    mkdir run.liftOChimp
    cd run.liftOChimp
    mkdir split out
    splitFile ../snp135ForLiftOver.bed 10000 split/chunk
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToPanTro3.over.chain.gz \
        \{check out exists out/panTro3.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    ssh swarm
    cd /hive/data/genomes/hg19/bed/snp135Ortho/run.liftOChimp
    para make jobList
#Completed: 4423 of 4423 jobs
#CPU time in finished jobs:     430555s    7175.92m   119.60h    4.98d  0.014 y
#IO & Wait Time:                 45877s     764.61m    12.74h    0.53d  0.001 y
#Average job time:                 108s       1.80m     0.03h    0.00d
#Longest finished job:             262s       4.37m     0.07h    0.00d
#Submission to last job:           542s       9.03m     0.15h    0.01d

    # Map coords to orangutan using liftOver.
    mkdir ../run.liftOPon
    cd ../run.liftOPon
    mkdir out
    ln -s ../run.liftOChimp/split .
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToPonAbe2.over.chain.gz \
        \{check out exists out/ponAbe2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    para make jobList
#Completed: 4423 of 4423 jobs
#CPU time in finished jobs:     591884s    9864.74m   164.41h    6.85d  0.019 y
#IO & Wait Time:                 55485s     924.74m    15.41h    0.64d  0.002 y
#Average job time:                 146s       2.44m     0.04h    0.00d
#Longest finished job:             380s       6.33m     0.11h    0.00d
#Submission to last job:          1403s      23.38m     0.39h    0.02d

    # Map coords to macaque using liftOver.
    mkdir ../run.liftOMac
    cd ../run.liftOMac
    mkdir out
    ln -s ../run.liftOChimp/split .
    cp /dev/null jobList
    foreach f (split/chunk*)
      echo liftOver $f \
        /hive/data/genomes/hg19/bed/liftOver/hg19ToRheMac2.over.chain.gz \
        \{check out exists out/rheMac2.$f:t.bed\} out/hg19.$f:t.unmapped \
        >> jobList
    end
    para make jobList
#Completed: 4423 of 4423 jobs
#CPU time in finished jobs:    1097552s   18292.53m   304.88h   12.70d  0.035 y
#IO & Wait Time:                 91301s    1521.69m    25.36h    1.06d  0.003 y
#Average job time:                 269s       4.48m     0.07h    0.00d
#Longest finished job:             697s      11.62m     0.19h    0.01d
#Submission to last job:          1555s      25.92m     0.43h    0.02d

    cd /hive/data/genomes/hg19/bed/snp135Ortho
    # Concatenate the chimp results, sorting by chimp pos in order to
    # efficiently access 2bit sequence in getOrthoSeq.  The output of
    # that is then sorted by the glommed human info field, so that we
    # can use join to combine chimp and macaque results in the next step.
    # Ditto for macaque and orangutan.  Each command pipe takes ~6 minutes:
    sort -k1,1 -k2n,2n run.liftOChimp/out/panTro3.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/panTro3/panTro3.2bit \
    | sort > panTro3.orthoGlom.txt
    sort -k1,1 -k2n,2n run.liftOPon/out/ponAbe2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/ponAbe2/ponAbe2.2bit \
    | sort > ponAbe2.orthoGlom.txt
    sort -k1,1 -k2n,2n run.liftOMac/out/rheMac2.chunk*.bed \
    | ~/kent/src/hg/snp/snpLoad/getOrthoSeq.pl /cluster/data/rheMac2/rheMac2.2bit \
    | sort > rheMac2.orthoGlom.txt
    wc -l panTro3.orthoGlom.txt ponAbe2.orthoGlom.txt rheMac2.orthoGlom.txt
#41804363 panTro3.orthoGlom.txt
#39856046 ponAbe2.orthoGlom.txt
#35918623 rheMac2.orthoGlom.txt

    # Use the glommed name field as a key to join up chimp and macaque
    # allele data.  Include glommed name from both files because if only
    # file 2 has a line for the key in 2.1, then 1.1 is empty.  Then plop
    # in the orthoGlom fields from each file, which are in the same order
    # as the chimp and macaque columns of snp135OrthoPanTro2RheMac2.
    join -o '1.1 2.1 1.2 1.3 1.4 1.5 1.6 2.2 2.3 2.4 2.5 2.6' \
      -a 1 -a 2 -e '?' \
      panTro3.orthoGlom.txt ponAbe2.orthoGlom.txt \
    | awk '{if ($1 != "?") { print $1, $3,$4,$5,$6,$7,$8,$9,$10,$11,$12; } \
            else           { print $2, $3,$4,$5,$6,$7,$8,$9,$10,$11,$12; }}' \
      > tmp.txt
    join -o '1.1 2.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 1.10 1.11 2.2 2.3 2.4 2.5 2.6' \
      -a 1 -a 2 -e '?' \
      tmp.txt rheMac2.orthoGlom.txt \
    | perl -wpe 'chomp; \
        ($glom12, $glom3, $o1Chr, $o1Start, $o1End, $o1Al, $o1Strand, \
         $o2Chr, $o2Start, $o2End, $o2Al, $o2Strand, \
         $o3Chr, $o3Start, $o3End, $o3Al, $o3Strand) = split; \
        $glomKey = ($glom12 ne "?") ? $glom12 : $glom3; \
        ($rsId, $hChr, $hStart, $hEnd, $hObs, $hAl, $hStrand) = \
          split(/\|/, $glomKey); \
        $o1Start =~ s/^\?$/0/;  $o2Start =~ s/^\?$/0/;  $o3Start =~ s/^\?$/0/; \
        $o1End   =~ s/^\?$/0/;  $o2End   =~ s/^\?$/0/;  $o3End   =~ s/^\?$/0/; \
        print join("\t", $hChr, $hStart, $hEnd, $rsId, $hObs, $hAl, $hStrand, \
                         $o1Chr, $o1Start, $o1End, $o1Al, $o1Strand, \
                         $o2Chr, $o2Start, $o2End, $o2Al, $o2Strand, \
                         $o3Chr, $o3Start, $o3End, $o3Al, $o3Strand) . "\n"; \
        s/^.*$//;' \
    | sort -k1,1 -k2n,2n > snp135OrthoPt3Pa2Rm2.bed
    hgLoadBed -tab -onServer -tmpDir=/data/tmp -renameSqlTable \
      -sqlTable=$HOME/kent/src/hg/lib/snpOrthoPanPonRhe.sql \
      hg19 snp135OrthoPt3Pa2Rm2 snp135OrthoPt3Pa2Rm2.bed
#Loaded 43184090 elements of size 22
    # Cleanup:
    rm -r run*/split tmp.txt *.orthoGlom.txt snp135Simple.bed 
    gzip snp135ExcludeIds.txt snp135ForLiftOver.bed &


############################################################################
# DBSNP CODING ANNOTATIONS (135) (DONE 11/14/11 angie)
# It wasn't necessary to redo this following the 9/1 re-run of doDbSnp.pl because
# that simply picked up new allele frequency info, no change to exceptions etc.
    cd /hive/data/outside/dbSNP/135/human
    # ncbiFuncAnnotations.txt has NCBI coords: 0-based, fully closed.
    # For anything except an insertion (0 bases between flanks),
    # we need to add 1 to the end coord.  For an insertion, we need
    # to add 1 to the start coord.  Make a hash of the insertion IDs,
    # then look up each ID in ncbiFuncAnnotations.txt to tell which
    # transform to apply.
    # Note: sort -u with the keys below is too restrictive -- we need full line uniq.
    zcat ncbiFuncAnnotations.txt.gz \
    | perl -we 'open($IDS, "zcat ncbiFuncInsertions.ctg.bed.gz |") || die "ids: $!"; \
              while (<$IDS>) { chomp; $ids{$_} = 1; } \
              close($IDS); \
              %coding = (2=>1, 3=>1, 4=>1, 8=>1, 9=>1, 41=>1, 42=>1, 44=>1, 45=>1); \
              while (<>) { \
                chomp;  @w = split("\t"); # id, ctg, start, end, ... \
                next unless $coding{$w[5]}; \
                $bed4 = join("\t", $w[1], $w[2], $w[3], $w[0]); \
                if (exists $ids{$bed4} && $w[3] == $w[2]+1) { \
                  $w[2]++; # 2-base insertions: increment start coord \
                } else { \
                  $w[3]++; # increment end coord to get half-open \
                } \
                print join("\t", @w) . "\n"; \
              }' \
    | sort -k1n,1n -k2,2 -k3n,3n -k5,5 -k6n,6n \
    | uniq \
      > ncbiCodingAnnotations.txt
    wc -l ncbiCodingAnnotations.txt
#2803490 ncbiCodingAnnotations.txt
    # How many & what kinds of function types?
    cut -f 6 ncbiCodingAnnotations.txt \
    | sort -n | uniq -c
# 512390 3   (coding-synon)
#1385793 8   (cds-reference -- ignored)
#  23909 41  (nonsense)
# 827675 42  (missense)
#  53703 44  (frameshift)
#     20 45  (cds-indel)
    # Gather up multiple annotation lines into one line per {snp, gene, frame}:
    perl -e  'while (<>) { chomp; \
                my ($rsId, $ctg, $s, $e, $txId, $fxn, $frm, $nt, $aa, $codon) = split("\t"); \
                if (defined $lastRs && \
                    ($lastRs != $rsId || $lastCtg ne $ctg || $lastS != $s || \
                     $lastTx ne $txId || $lastFrm ne $frm)) { \
                  if (defined $refRow) { \
                    $fxns = "$refRow->[0],$fxns";  $nts = "$refRow->[1],$nts"; \
                    $aas = "$refRow->[2],$aas";    $codons = "$refRow->[3],$codons"; \
                  } \
                  print "$lastCtg\t$lastS\t$lastE\trs$lastRs\t$lastTx\t$lastFrm\t" . \
                        "$count\t$fxns\t$nts\t$codons\t$aas\n"; \
                  $refRow = undef;  @rows = ();  ($count, $fxns, $nts, $codons, $aas) = (); \
                } \
                ($lastRs, $lastCtg, $lastS, $lastE, $lastTx, $lastFrm) = \
                    ($rsId, $ctg, $s, $e, $txId, $frm); \
                $count++; \
                if ($fxn == 8) { \
                  $refRow = [$fxn, $nt, $aa, $codon]; \
                } else { \
                 $fxns .= "$fxn,";  $nts .= "$nt,";  $aas .= "$aa,";  $codons .= "$codon,"; \
                } \
              } \
              if (defined $refRow) { \
                $fxns = "$refRow->[0],$fxns";  $nts = "$refRow->[1],$nts"; \
                $aas = "$refRow->[2],$aas";    $codons = "$refRow->[3],$codons"; \
              } \
              print "$lastCtg\t$lastS\t$lastE\trs$lastRs\t$lastTx\t$lastFrm\t" . \
                    "$count\t$fxns\t$nts\t$codons\t$aas\n";' \
      ncbiCodingAnnotations.txt \
    | liftUp snp135CodingDbSnp.bed /hive/data/genomes/hg19/jkStuff/liftContigs.lft warn stdin
    hgLoadBed hg19 snp135CodingDbSnp -sqlTable=$HOME/kent/src/hg/lib/snp125Coding.sql \
      -renameSqlTable -tab -notItemRgb -allowStartEqualEnd \
      snp135CodingDbSnp.bed
#Loaded 1385812 elements of size 11


############################################################################
# SNPMASKED SEQUENCE FOR SNP135 (DONE 11/14/11 angie)
    mkdir /hive/data/genomes/hg19/snp135Mask
    cd /hive/data/genomes/hg19/snp135Mask
    # Identify rsIds with various problems -- we will exclude those.
    zcat /hive/data/outside/dbSNP/135/human/snp135.bed.gz \
    | awk '$18 ~ /MultipleAlignments|ObservedTooLong|ObservedWrongFormat|ObservedMismatch|MixedObserved/ {print $4;}' \
      | sort -u \
      > snp135ExcludeRsIds.txt
    zcat /hive/data/outside/dbSNP/135/human/snp135.bed.gz \
    | grep -vFwf snp135ExcludeRsIds.txt \
      > snp135Cleaned.bed
    wc -l snp135Cleaned.bed
#49922101 snp135Cleaned.bed

    # Substitutions:
    mkdir substitutions
    snpMaskSingle snp135Cleaned.bed /hive/data/genomes/hg19/hg19.2bit stdout diffObserved.txt \
    | faSplit byname stdin substitutions/
#Masked 44283699 snps in 44281659 out of 3131050506 genomic bases
#/hive/data/genomes/hg19/hg19.2bit has 3137161264 total bases, but the total number of bases in sequences for which we masked snps is 3131050506 (difference is 6110758)
    # warnings about differing observed strings at same base position:
    wc -l diffObserved.txt
#3661 diffObserved.txt
#TODO: send list to dbSNP.
    # Check that 6110758 is the total #bases in sequences with nothing in snp135Cleaned:
    grep -Fw single snp135Cleaned.bed | cut -f 1 | uniq > /data/tmp/1
    grep -vwf /data/tmp/1 ../chrom.sizes \
    | awk 'BEGIN {TOTAL = 0;}  {TOTAL += $2;}  END {printf "%d\n", TOTAL;}'
#6110758
    # Make sure that sizes are identical, first diffs are normal -> IUPAC,
    # and first diffs' case is preserved:
    foreach f (substitutions/chr*.fa)
      faCmp -softMask $f ../[1-9UMXY]*/$f:t |& grep -v "that differ"
    end
#chr1 in substitutions/chr1.fa differs from chr1 at ../1/chr1.fa at base 10233 (y != c)
#chr10 in substitutions/chr10.fa differs from chr10 at ../10/chr10.fa at base 60522 (K != T)
#...
#(output OK -- ambiguous bases replacing [agct] at SNP positions)
    foreach f (substitutions/chr*.fa)
      echo $f:t:r
      mv $f $f:r.subst.fa
      gzip $f:r.subst.fa &
    end

    # Insertions & deletions not done.  To date we have only offered substs for download.
    # If there is user demand, use template from snp131 above.

    # Clean up and prepare for download:
    gzip snp135Cleaned.bed &
    foreach d (substitutions)
      pushd $d
        md5sum *.gz > md5sum.txt
        cp /hive/data/genomes/hg19/snp132Mask/$d/README.txt .
      popd
    end
    # Edit the README.txt.

    # Create download links on hgwdev.
    mkdir /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/snp135Mask
    ln -s /hive/data/genomes/hg19/snp135Mask/substitutions/* \
      /usr/local/apache/htdocs-hgdownload/goldenPath/hg19/snp135Mask/


#############################################################################
# SIB Transcriptome (DONE 2011-12-02 Chin)

    # Create working directory and download data from where Christian
    # Iseli (Christian.Iseli at licr.org) put it, and unpack. 
    mkdir -p /hive/data/outside/lirc
    cd /hive/data/outside/lirc
    wget --timestamping ftp://ftp.licr.org/pub/hg19/HTr.gtf.gz 
    wget --timestamping ftp://ftp.licr.org/pub/hg19/txg.tar.gz 
 
    cd /hive/data/genomes/hg19/bed/
    mkdir sibTranscriptome
    cd sibTranscriptome
    tar -zxvf /hive/data/outside/lirc/txg.tar.gz
    cp /hive/data/outside/lirc/HTr.gtf.gz .

    zcat HTr.gtf.gz | ldHgGene hg19 sibGene stdin
    # Reading stdin
    # Read 195300 transcripts in 2564421 lines in 1 files
    # 195300 groups 25 seqs 1 sources 2 feature types
    # 195300 gene predictions

    # Do a little data cleanup and transformation and load splice graphs
    # into database.
    sed 's/altGraphX/sibTxGraph/' ~/kent/src/hg/lib/altGraphX.sql > sibTxGraph.sql
    cat txg/*.txg | txgToAgx stdin stdout | hgLoadBed -notItemRgb \
      -sqlTable=sibTxGraph.sql hg19 sibTxGraph stdin
    # Reading stdin
    # Loaded 46973 elements of size 18
    # Sorted
    # Creating table definition for sibTxGraph
    # Saving bed.tab
    # Loading hg19

    # Create sibAltEvents track for analysed alt-splices.
    # Not on RR for hg18 and hg19, so do not push it out
   cat txg/*.txg | txgAnalyze stdin /cluster/data/hg19/hg19.2bit sibAltEvents.bed
   awk '$2 >= 0' sibAltEvents.bed | sort | uniq > foo.bed
   hgLoadBed hg19 sibAltEvents foo.bed
    # Reading foo.bed
    # Loaded 431590 elements of size 6
    # Sorted
    # Creating table definition for sibAltEvents
    # Saving bed.tab
    # Loading hg19


    # push sibGene and sibTxGraph for hg19


############################################################################
