# for emacs: -*- mode: sh; -*-


# This file describes how we made the browser database on 
# NCBI build 35 (May 2004 freeze)

# HOW TO BUILD AN ASSEMBLY FROM NCBI FILES 
# ---------------------------------------

    # Make gs.18 directory, gs.18/build35 directory, and gs.18/ffa directory.
    ssh eieio
    mkdir /cluster/store5/gs.18
    mkdir /cluster/store5/gs.18/build35
    mkdir /cluster/store5/gs.18/agp
    mkdir /cluster/store5/gs.18/ffa

    #    Make a symbolic link from /cluster/store1 to this location
    #	(I assume there is some use for this later ?)
	
    cd /cluster/store1
    ln -s /cluster/store5/gs.18 ./gs.18
    ln -s /cluster/store5/gs.18/build35 /cluster/data/hg17

    #    Make a symbolic link from your home directory to the build dir:
    #	(Investigate what this is used for, may no longer be necessary)

    ln -s /cluster/store5/gs.18/build35 ~/oo

# NCBI download site, fetch everything into this one directory:

    #	with the machine and password in your $HOME/.netrc file, this
    #	wget command will require no login.  Your $HOME/.netrc file
    #	is set to 'chmod 600 .netrc' to prevent anyone from finding
    #	the data.  (There were some early files that later moved
    #		into an OLD subdirectory.  They were broken.)
    mkdir /cluster/store5/gs.18/ncbi
    cd /cluster/store5/gs.18/ncbi
    wget --timestamping ftp://ftp.ncbi.nlm.nih.gov/build_35/*

    # FYI: agp file format documented at:
    #	http://www.ncbi.nlm.nih.gov/Genbank/WGS.agpformat.html    # fixup a couple of names for our own purposes here
    cd /cluster/store5/gs.18/agp
    ln -s ../ncbi/chr*.agp ../ncbi/chr*.fa.gz .
    sed -e "s#MT/NC_001807.4#NC_001807#" ../ncbi/chrMT.agp > chrM.agp
    sed -e "s/NG_002392.2/NG_002392/" ../ncbi/DR52.agp > chr6_hla_hap1.agp
    sed -e "s/NG_002433.1/NG_002433/" ../ncbi/DR53.agp > chr6_hla_hap2.agp
    zcat ../ncbi/DR52.fa.gz | \
	sed -e "s/gi|29124352|ref|NG_002392.2/ref|NG_002392/" | \
	gzip > chr6_hla_hap1.fa.gz
    zcat ../ncbi/DR53.fa.gz | \
	sed -e "s/gi|28212470|ref|NG_002433.1/ref|NG_002433/" | \
	gzip > chr6_hla_hap2.fa.gz
    zcat ../ncbi/chrMT.fa.gz | \
	sed -e "s/gi|17981852|ref|NC_001807.4/ref|NC_001807/" | \
	gzip > chrM.fa.gz
    

    #  Put all the agp files together into one.
    cd /cluster/store5/gs.18/build35
    #	The chrM sequence now has its own agp, remove it from
    #	ref_placed.agp
    sed -e "/^NC_001807/d" ../ncbi/ref_placed.agp > ref_placed.agp
    cat ref_placed.agp ../agp/chrM.agp ../ncbi/ref_unplaced.agp \
	../agp/chr6_hla_hap1.agp ../agp/chr6_hla_hap2.agp \
	../ncbi/PAR.agp > ncbi_build35.agp

    #	and into ffa
    cd /cluster/store5/gs.18/ffa
    #	There is a single bogus line at the end of ref_placed.fa.gz
    #	declaring the NC_001807 MT sequence, this was later replaced by
    #	chrMT.fa.gz, so remove that one line:
    zcat ../ncbi/ref_placed.fa.gz | sed -e "/^>ref|NC_001807/d" | \
	gzip > ref_placed.fa.gz
    #	(That's a 40 minute job)

    #	sequence.inf is usually here, symlink it
    ln -s ../ncbi/sequence.inf
    #	put all the fa.gz files together in one big fa.gz
    time zcat ref_placed.fa.gz ../agp/chrM.fa.gz ../ncbi/ref_unplaced.fa.gz \
	../agp/chr6_hla_hap?.fa.gz ../ncbi/PAR.fa.gz | gzip \
	> ncbi_build35.fa.gz
    #	real    37m42.208s
    #	user    37m3.490s
    #	sys     0m31.430s

    #	Make a listing of all the fasta record headers, just FYI:
    cd /cluster/store5/gs.18/ffa
    zcat ffa/ncbi_build35.fa.gz | grep "^>" > ncbi.fa.headers


    #	New to this build is the sequence: NC_001807 which is the
    #	mitochondria sequence.  This prefix NC_ is new to the process
    #	and will have to be accounted for below.  The other two special
    #	prefixes are similar to what was seen before:
    #	from DR52.agp NG_002392
    #	Homo sapiens major histocompatibility complex, class II,
    #		DR52 haplotype (DR52) on chromosome 6
    #	and from DR53.agp NG_002433
    #	Homo sapiens major histocompatibility complex, class II,
    #		DR53 haplotype (DR53) on chromosome 6

    #	Fixup seq_contig.md
    #
    #	It has a bunch of stuff belonging to the Celera
    #	genome assembly.  Filter those out.  I don't know what the
    #	NT_07959[0-7] items are, but there are no definitions for them
    #	in the agp files and no sequence in any fa.gz file.
    #	Fixup the names for the NG_ items, and change chrom MT to be M
    cd /cluster/store5/gs.18/build35
    egrep -v "Celera|NT_07959[0-7]" ../ncbi/seq_contig.md | \
	sed -e "s/6|NG_002392/6_hla_hap1/" \
	-e "s/6|NG_002433/6_hla_hap2/" \
	-e "s/^9606\tMT|NC_001807/9606\tM/" \
	> temp_contig.md

    #	get the randoms sorted in proper order.  The createNcbiLifts
    #	does not work correctly if the randoms are not grouped together
    #	by chromosome
    grep -v "|" temp_contig.md  > seq_contig.md
    #	This pulls out all the randoms and groups them within the
    #	same chrom but leaving them in the same order as they orginally
    #	were  (warning this is BASH code ...)
    grep "|" temp_contig.md | awk -F"|" '{print $1}' | \
        awk '{print $2}' | sort -n -u | while read CHR
do
        grep "[^0-9]${CHR}|" temp_contig.md
done >> seq_contig.md


    # Sanity check, checkYbr was updated to handle the NC_ identifier
    time zcat ../ffa/ncbi_build35.fa.gz | $HOME/bin/i386/checkYbr \
	ncbi_build35.agp stdin seq_contig.md > check.seq_contig
    #	real    2m34.143s
    #	user    2m24.970s
    #	sys     0m8.900s
    #	result should be clean:
    cat check.seq_contig
    #	Read 380 contigs from ncbi_build35.agp
    #	Verifying sequence sizes in stdin
    #	0 problems detected


    # Convert fa files into UCSC style fa files and place in "contigs"
    # directory inside the gs.18/build35 directory 
    #	(a check that can be done here is make a list of the contigs
    #	in this ./contigs directory before and compare it with the
    #	list of distributed contigs created after they have been
    #	disbursed.)
    #	faNcbiToUcsc was fixed to handle the NC_ identifier

    cd /cluster/store5/gs.18/build35
    #	We've been through this often
    mv contigs contigs.0
    time zcat ../ffa/ncbi_build35.fa.gz | $HOME/bin/i386/faNcbiToUcsc \
	-split -ntLast stdin contigs
    #	real    5m10.938s
    #	user    2m20.070s
    #	sys     0m51.020s
    #	If you want to compare anything to previous work, check now, then:
    rm -fr contigs.0

    # Determine the chromosome sizes from agps
    #	Watch carefully how chrY gets constructed.  I'm not sure
    #	this chrom_sizes represents the whole length of chrY with
    #	the PAR added.  We will see about that.
    #	Script updated to handle new chrom names:
    #	my @chroms = (1 .. 22, 'X', 'Y', 'M', '6_hla_hap1', '6_hla_hap2');

    cd /cluster/store5/gs.18/build35
    /cluster/bin/scripts/getChromSizes ../agp
    #	Create chrom.lst list for use in foreach() loops
    awk '{print $1}' chrom_sizes | sed -e "s/chr//" > chrom.lst

    # Create lift files (this will create chromosome directory structure) and
    #	inserts file
  
    /cluster/bin/scripts/createNcbiLifts -s chrom_sizes seq_contig.md .

    # Create contig agp files (will create contig directory structure)
	
    /cluster/bin/scripts/createNcbiCtgAgp seq_contig.md ncbi_build35.agp .

    # Create chromsome random agp files.

    /cluster/bin/scripts/createNcbiChrAgp -randomonly .

    # Copy the original chrN.agp files from the gs.18/agp directory 
    #    into each of the chromosome directories since they contain better 
    #    gap information. Delete the comments at top from these.
    cd /cluster/store5/gs.18/build35
    foreach c ( `cat chrom.lst` )
	sed -e "/^#.*/d" ../agp/chr${c}.agp > ./${c}/chr${c}.agp
    end
    #	chrM needs a name fixup
    sed -e "s#NC_001807#chrM#" ../agp/chrM.agp > M/chrM.agp

    # Distribute contig .fa to appropriate directory (assumes all files
    # are in "contigs" directory).

    # Create inserts file from agp and lift files (new - added by Terry, 2004-07-12)
    /cluster/bin/scripts/createInserts /cluster/data/hg17 > /cluster/data/hg17/inserts

    # create global data link for everyone.  No more home directory
    # links required.
    ln -s /cluster/store5/gs.18/build35 /cluster/data/hg17
    cd /cluster/data/hg17
    /cluster/bin/scripts/distNcbiCtgFa contigs .
    #	Verify that everything was moved properly, the contigs directory
    #	should be empty:
    ls contigs
    #	Nothing there, then remove it
    rmdir  contigs

    #	Make a list of the contigs for use later
    rm contig.lst
    touch contig.lst
    foreach chrom ( `cat chrom.lst` )
	foreach c ( $chrom/N{C,G,T}_?????? )
	    set contig = $c:t
	    echo "${chrom}/${contig}/${contig}.fa" >> contig.lst
	end
    end
    #   For later comparisons, this is how many contigs we have:
    wc -l contig.lst
    #	380

    #	Note 2004-06-30 - there are some clone numbers left in some of
    #	the NCBI files that are incorrect.  Due to version number
    #	changes, more than one version is listed.  Namely for accession
    #	numbers: AC004491 AC004921 AC004983 AC005088 AC006014 AC099654
    #	The AGP files are correct, the sequence.inf file lists these
    #	twice: AC004491.1 AC004491.2
    #	AC004921.1 AC004921.2 AC004983.2 AC004983.3
    #	AC005088.2 AC005088.3 AC006014.2 AC006014.3
    #	AC099654.4 AC099654.5 

    # FILES ARE NOW READY FOR REPEAT MASKING - start that process as
    #	other steps here can proceed in parallel.

    #	Previous practice used to copy everything over for jkStuff from a
    #	previous build.  Rather than do that, pick up whatever is needed
    #	at the time it is needed and verify that it is going to do what
    #	you expect.

    cd /cluster/data/hg17
    mkdir jkStuff

    # Create the contig.gl files - XXX - NCBI doesn't deliver
    # contig_overlaps.agp - 2004-06-18 - this is beginning to come
    # together and there is now a contig_overlaps.agp file

    #	This is properly done below with a combination of psLayout
    #	alignments to create the contig_overlaps.agp file
    # /cluster/bin/i386/agpToGl contig_overlaps.agp . -md=seq_contig.md
    # Create chromosome gl files
    # jkStuff/liftGl.csh contig.gl

# CREATING DATABASE  (DONE - 2004-05-20 - Hiram)
    #	RE-DONE for new NIBS - 2004-06-03
    ssh hgwdev
    # Make sure there is at least 5 gig free on hgwdev:/var/lib/mysql
    df -h /var/lib/mysql
#	Filesystem            Size  Used Avail Use% Mounted on
#	/dev/sdc1             1.8T  303G  1.4T  19% /var/lib/mysql

    # Create the database.
    hgsql -e 'create database hg17' mysql
    # Copy over grp table (for track grouping) from another database:
    hgsql -e "create table grp (PRIMARY KEY(NAME)) select * from hg16.grp" hg17

    # ENCODE groups
    # Added 2005-08016 kate
    echo 'UPDATE grp SET priority=7 WHERE name="varRep"'| hgsql hg17
    echo 'UPDATE grp SET priority=8 WHERE name="encode"'| hgsql hg17
    echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeGenes", "ENCODE Regions and Genes", 8.1)' | hgsql hg17
    echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeTxLevels", "ENCODE Transcript Levels", 8.2)' | hgsql hg17
    echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeChip", "ENCODE Chromatin Immunoprecipitation", 8.3)' | hgsql hg17
    echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeChrom", "ENCODE Chromosome, Chromatin and DNA Structure", 8.4)' | hgsql hg17
    echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeCompGeno", "ENCODE Comparative Genomics", 8.5)' | hgsql hg17
    echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeVariation", "ENCODE Variation", 8.6)' | hgsql hg17
    echo 'INSERT INTO grp (name, label, priority) VALUES ("encodeAnalysis", "ENCODE Analysis", 8.9)' | hgsql hg17
    

# MAKE CHROMINFO TABLE WITH (TEMPORARILY UNMASKED) NIBS
#	(DONE - 2004-05-21 - Hiram)
    #	RE-DONE with new NIBS - 2004-06-03
    # Make nib/, unmasked until RepeatMasker and TRF steps are done.
    # Do this now so that the chromInfo table will exist and thus the
    #	trackDb tables can be built in the next step.
    #	These unmasked nibs will be replaced by the masked nibs after
    #	repeat mask and trf are done.
    ssh eieio
    cd /cluster/data/hg17
    # Make chr*.fa from contig .fa
    #  Copied chrFa.sh from hg16/jkStuff, renamed it to chrFa.csh
    time ./jkStuff/chrFa.csh
    #	real    13m24.710s
    #	user    9m0.360s
    #	sys     1m15.820s

    mkdir nib
    foreach c (`cat chrom.lst`)
      foreach f ($c/chr${c}{,_random}.fa)
        if (-e $f) then
          echo "nibbing $f"
          /cluster/bin/i386/faToNib $f nib/$f:t:r.nib
        endif
      end
    end

    # Make symbolic links from /gbdb/hg17/nib to the real nibs.
    ssh hgwdev
    mkdir -p /gbdb/hg17/nib
    ln -s /cluster/data/hg17/nib/chr*.nib /gbdb/hg17/nib
    # Load /gbdb/hg17/nib paths into database and save size info.
    cd /cluster/data/hg17
    hgsql hg17  < $HOME/kent/src/hg/lib/chromInfo.sql
    hgNibSeq -preMadeNib hg17 /gbdb/hg17/nib */chr*.fa
    hgsql -N -e "select chrom,size from chromInfo order by chrom" hg17 \
	> chrom.sizes
    # You can compare this chrom.sizes with the previously created
    # chrom_sizes.  Should be no difference
    sort chrom_sizes > s0
    sort chrom.sizes | grep -v random > s1
    diff s0 s1
    rm s0 s1

# MAKE HGCENTRALTEST ENTRY AND TRACKDB TABLE (DONE - 2004-05-21 - Hiram)
    #	dbDb orderKey updated 2004-06-08 - Hiram
    ssh hgwdev
    #	reset dbDb orderKey - these have never been ordered properly
    #	before, this will get them on the program.
    hgsql -e 'update dbDb set orderKey=11 where name = "hg16";' \
	-h genome-testdb hgcentraltest
    hgsql -e 'update dbDb set orderKey=12 where name = "hg15";' \
	-h genome-testdb hgcentraltest
    hgsql -e 'update dbDb set orderKey=13 where name = "hg13";' \
	-h genome-testdb hgcentraltest

    # Enter hg17 into hgcentraltest.dbDb so test browser knows about it:
    hgsql -e 'INSERT INTO dbDb (name, description, nibPath, organism, \
	defaultPos, active, orderKey, genome, scientificName, \
	htmlPath, hgNearOk, hgPbOk, sourceName) \
	VALUES("hg17", "May 2004", "/gbdb/hg17/nib", "Human", \
	"chr4:56214201-56291736", 1, 10, "Human", "Homo sapiens", \
	"/gbdb/hg17/html/description.html", 0, 0, "NCBI Build 35");' \
	-h genome-testdb hgcentraltest
    # Make trackDb table so browser knows what tracks to expect:
    cd ~/kent/src/hg/makeDb/trackDb
    cvs up -d -P .
    # Edit the makefile to add hg17 in all the right places and do
    make update
    make alpha
    cvs commit makefile

# MAKE LIFTALL.LFT, NCBI.LFT (DONE - 2004-05-21 - Hiram)
    #	Re-DONE with new randoms - 2004-06-03 - Hiram)
    cd /cluster/data/hg17
    mkdir -p jkStuff
    cat */lift/{ordered,random}.lft > jkStuff/liftAll.lft
    # Create jkStuff/ncbi.lft for lifting stuff built with the NCBI assembly.
    # Note: this ncbi.lift will not lift floating contigs to chr_random coords,
    # but it will show the strand orientation of the floating contigs 
    # (grep for '|').
    #   mdToNcbiLift seq_contig.md jkStuff/ncbi.lft 
    #	XXXX - appears to be unused, not done - Hiram

# REPEAT MASKING (DONE - 2004-05-24 - Hiram)
    #	The randoms were rearranged after this was first done,
    #	they are re-made below 2004-06-02)
    #	Record the RM version here:
    #	RepBase Update 8.12, RM database version 20040130
    #	as this changes over time and there is no record in the results

    # Split contigs, run RepeatMasker, lift results

    #	This split takes about 8 minutes
    ssh eieio
    cd /cluster/data/hg17
    foreach chrom ( `cat chrom.lst` )
	foreach c ( $chrom/N{C,G,T}_?????? )
	    set contig = $c:t
	    echo "splitting ${chrom}/${contig}/${contig}.fa"
	    faSplit size ${chrom}/${contig}/$contig.fa 500000 \
		${chrom}/${contig}/${contig}_ \
		-lift=${chrom}/${contig}/$contig.lft -maxN=500000
	end
    end

    #- Make the run directory and job list:
    cd /cluster/data/hg17
    mkdir -p jkStuff
    #  According to RepeatMasker help file, no arguments are required to
    #	specify species because its default is set for primate (human)
    #  This run script saves the .tbl file to be sent to Arian.  He uses
    # those for his analysis.  Sometimes he needs the .cat and .align files for
    # checking problems.  Krish needs the .align files, they are large.

    cat << '_EOF_' > jkStuff/RMHuman
#!/bin/csh -fe

cd $1
pushd .
/bin/mkdir -p /tmp/hg17/$2
/bin/cp $2 /tmp/hg17/$2/
cd /tmp/hg17/$2
/cluster/bluearc/RepeatMasker/RepeatMasker -ali -s $2
popd
/bin/cp /tmp/hg17/$2/$2.out ./
 if (-e /tmp/hg17/$2/$2.align) /bin/cp /tmp/hg17/$2/$2.align ./
if (-e /tmp/hg17/$2/$2.tbl) /bin/cp /tmp/hg17/$2/$2.tbl ./
# if (-e /tmp/hg17/$2/$2.cat) /bin/cp /tmp/hg17/$2/$2.cat ./
/bin/rm -fr /tmp/hg17/$2/*
/bin/rmdir --ignore-fail-on-non-empty /tmp/hg17/$2
/bin/rmdir --ignore-fail-on-non-empty /tmp/hg17
'_EOF_'
    # << this line makes emacs coloring happy
    chmod +x jkStuff/RMHuman

    ssh eieio
    cd /cluster/data/hg17
    mkdir RMRun
    rm -f RMRun/RMJobs
    touch RMRun/RMJobs
    foreach d ( `cat chrom.lst` )
     foreach c ( ${d}/N{C,G,T}_*/N{C,G,T}_*_*.fa )
        set f = $c:t
        set cc = $c:h
        set contig = $cc:t
        echo /cluster/store5/gs.18/build35/jkStuff/RMHuman \
   		/cluster/store5/gs.18/build35/${d}/${contig} $f \
   '{'check out line+ /cluster/store5/gs.18/build35/${d}/${contig}/$f.out'}' \
          >> RMRun/RMJobs
      end
    end

    # We have 5971 jobs in RMJobs:
    wc RMRun/RMJobs
    #	5970   41790 1105804 RMRun/RMJobs

    #- Do the run
    ssh kk
    cd /cluster/data/hg17/RMRun
    para create RMJobs
    para try, para check, para check, para push, para check,...

    #- While that is running, you can run TRF (simpleRepeat) on the small
    # cluster.  See SIMPLE REPEAT section below
# Completed: 5970 of 5970 jobs
# CPU time in finished jobs:   45189516s  753158.60m 12552.64h  523.03d  1.433 y
# IO & Wait Time:                141333s    2355.55m    39.26h    1.64d  0.004 y
# Average job time:                7593s     126.55m     2.11h    0.09d
# Longest job:                    10268s     171.13m     2.85h    0.12d
# Submission to last job:         81484s    1358.07m    22.63h    0.94d

    #	Lift up the split-contig .out's to contig-level .out's
    #
    #	If a mistake is made in the following it would be possible to
    #	destroy all the RM output.  So, just to be paranoid, save all
    #	the RM output in bluearc for the time being:
    ssh eieio

    cd /cluster/data/hg17
    mkdir /cluster/bluearc/hg17/RMOutput
    foreach c ( `cat chrom.lst` )
     foreach d ( ${c}/N{C,G,T}_* )
	set T = /cluster/bluearc/hg17/RMOutput/${d}
	mkdir -p ${T}
        cd ${d}
        set contig = $d:t
        cp -p ${contig}_?{,?,??}.fa.out ${T}
        cd ../..
	echo "${d} done"
     end
    end
    #	Make sure we got them all:
    #	(this doesn't work later since there are more *.fa.out files
    #	after the lifting.  More explicitly to find just these:
    #		find . -name "N?_*_*.fa.out" -print | wc -l
    find . -name "*.fa.out" -print | wc -l
    #	5970
    find /cluster/bluearc/hg17/RMOutput -type f | wc -l
    #	5970
    #	same count

    #	OK, now you can try this operation, do it in a script like this
    #	and save the output of the script for a record of what happened.

    cat << '_EOF_' > jkStuff/liftRM.csh
#!/bin/csh -fe
foreach c ( `cat chrom.lst` )
 foreach d ( ${c}/N{C,G,T}_* )
    cd $d
    set contig = $d:t
    liftUp $contig.fa.out $contig.lft warn ${contig}_?{,?,??}.fa.out 
    cd ../..
 end
end
'_EOF_'
    chmod +x jkStuff/liftRM.csh
    mkdir scriptsOutput
    time jkStuff/liftRM.csh > scriptsOutput/liftRM.1 2>&1
    #	real    4m37.572s
    #	user    1m19.130s
    #	sys     0m32.950s
    #	Check that they all were done:
    grep "fa.out" scriptsOutput/liftRM.1 | wc -l
    #	5959
    #	same count as above

    #- Lift up RepeatMask .out files to chromosome coordinates via
    # picked up jkStuff/liftOut2.sh from the hg16 build.  Renamed to
    # liftOut2.csh, changed the line that does the chrom listing

    time ./jkStuff/liftOut2.csh > scriptsOutput/liftOut2 2>&1
    #	real    9m46.780s
    #	user    1m18.900s
    #	sys     7m33.990s

    #- By this point, the database should have been created (above):
    ssh hgwdev
    cd /cluster/data/hg17
    time hgLoadOut hg17 ?/*.fa.out ??/*.fa.out 6_hla_hap?/*.fa.out > \
	scriptsOutput/hgLoadOut 2>&1
    #	real    5m59.137s
    #	user    1m47.550s
    #	sys     0m15.410s

    # errors during this load:  (there are always a couple of these)
    #	Strange perc. field -6.1 line 243543 of 2/chr2.fa.out
    #	Strange perc. field -5.6 line 243543 of 2/chr2.fa.out
    #	Strange perc. field -6.1 line 243545 of 2/chr2.fa.out
    #	Strange perc. field -5.6 line 243545 of 2/chr2.fa.out
    #	Strange perc. field -0.2 line 30322 of 17/chr17.fa.out
    #	Strange perc. field -0.2 line 30324 of 17/chr17.fa.out
    #	Strange perc. field -0.2 line 30326 of 17/chr17.fa.out
    #	Strange perc. field -0.2 line 30328 of 17/chr17.fa.out
    #	Strange perc. field -18.6 line 77034 of 19/chr19.fa.out

    #	Verify we have similar results to previous assembly:
    #	featureBits hg17 rmsk
    #	1391378842 bases of 2867328468 (48.525%) in intersection
    #	featureBits hg16 rmsk
    #	1388770568 bases of 2865248791 (48.469%) in intersection
    #	Now proceed to MASK SEQUENCE BOTH REPEATMASKER AND SIMPLE REPEAT/TRF
    #	following the SIMPLE REPEAT sections below

# Re-Running REPEAT_MASKER on the new Randoms (DONE - 2004-06-02 - Hiram)
    ssh eieio
    cd /cluster/data/hg17
    grep "|" seq_contig.md | awk '{print $2}' | sed -e "s#|#/#" > randoms.lst

    mkdir /cluster/data/hg17/RMRandoms
    foreach r ( `cat randoms.lst` )
	set d = $r:h
	set contig = $r:t
	foreach c ( ${r}/N{C,G,T}_*_*.fa )
	    set f = $c:t
	    echo /cluster/store5/gs.18/build35/jkStuff/RMHuman \
   		/cluster/store5/gs.18/build35/${d}/${contig} $f \
   '{'check out line+ /cluster/store5/gs.18/build35/${d}/${contig}/$f.out'}' \
          >> RMRandoms/RMJobs
	end
    end

    ssh kk
    cd /cluster/data/hg17/RMRandoms
    para create RMJobs
    para try, para check, para check, para push, para check,...
# Completed: 94 of 94 jobs
# CPU time in finished jobs:     221454s    3690.91m    61.52h    2.56d  0.007 y
# IO & Wait Time:                   866s      14.43m     0.24h    0.01d  0.000 y
# Average job time:                2365s      39.42m     0.66h    0.03d
# Longest job:                     9062s     151.03m     2.52h    0.10d
# Submission to last job:          9106s     151.77m     2.53h    0.11d

    #	Continuing with the paranoia theme, let's backup all the RM output
    #
    ssh eieio

    cd /cluster/data/hg17
    mkdir /cluster/bluearc/hg17/RMRandoms
    foreach c ( `cat chrom.lst` )
     foreach d ( ${c}/N{C,G,T}_* )
	set T = /cluster/bluearc/hg17/RMRandoms/${d}
	mkdir -p ${T}
        cd ${d}
        set contig = $d:t
        cp -p ${contig}_?{,?,??}.fa.out ${T}
        cd ../..
	echo "${d} done"
     end
    end
    #	Make sure we got them all:
    find . -name "N?_*_*.fa.out" -print | wc -l
    #	5959
    find /cluster/bluearc/hg17/RMRandoms -type f | wc -l
    #	5959
    #	same count


    time jkStuff/liftRM.csh > scriptsOutput/liftRM2.1 2>&1
    #	real    4m46.302s
    #	user    1m18.260s
    #	sys     0m18.000s
    #	Check that they all were done:
    grep "fa.out" scriptsOutput/liftRM2.1 | wc -l
    #	5959
    #	same count as above

    #- Lift up RepeatMask .out files to chromosome coordinates via
    # picked up jkStuff/liftOut2.sh from the hg16 build.  Renamed to
    # liftOut2.csh, changed the line that does the chrom listing

    time ./jkStuff/liftOut2.csh > scriptsOutput/liftOut2.1 2>&1
    #	real    2m46.347s
    #	user    1m18.650s
    #	sys     0m15.990s

    #- By this point, the database should have been created (above):
    ssh hgwdev
    cd /cluster/data/hg17
    time hgLoadOut hg17 ?/*.fa.out ??/*.fa.out 6_hla_hap?/*.fa.out > \
	scriptsOutput/hgLoadOut 2>&1
    #	real    5m59.137s
    #	user    1m47.550s
    #	sys     0m15.410s

    # errors during this load:  (there are always a couple of these)
    #	Strange perc. field -6.1 line 243543 of 2/chr2.fa.out
    #	Strange perc. field -5.6 line 243543 of 2/chr2.fa.out
    #	Strange perc. field -6.1 line 243545 of 2/chr2.fa.out
    #	Strange perc. field -5.6 line 243545 of 2/chr2.fa.out
    #	Strange perc. field -0.2 line 30322 of 17/chr17.fa.out
    #	Strange perc. field -0.2 line 30324 of 17/chr17.fa.out
    #	Strange perc. field -0.2 line 30326 of 17/chr17.fa.out
    #	Strange perc. field -0.2 line 30328 of 17/chr17.fa.out
    #	Strange perc. field -18.6 line 77034 of 19/chr19.fa.out

    #	Verify we have similar results to previous assembly:
    #	featureBits hg17 rmsk
    #	1390952984 bases of 2866216770 (48.529%) in intersection
    #	featureBits hg17 rmsk  #with previous randoms:
    #	1391378842 bases of 2867328468 (48.525%) in intersection
    #	featureBits hg16 rmsk
    #	1388770568 bases of 2865248791 (48.469%) in intersection
    #	Now proceed to MASK SEQUENCE BOTH REPEATMASKER AND SIMPLE REPEAT/TRF
    #	following the SIMPLE REPEAT sections below

# SIMPLE REPEAT [TRF] TRACK (DONE - 2004-05-21 - Hiram)
    #	Re-done with new randoms, 2004-06-02 - Hiram
    #	Copy the contigs, first to the bluearc, then to /iscratch/i
    ssh eieio
    mkdir /cluster/bluearc/hg17
    mkdir /cluster/bluearc/hg17/contigs

    cd /cluster/data/hg17
    foreach ctg ( `cat contig.lst` )
	set c = $ctg:t
 	echo "$ctg > /cluster/bluearc/hg17/contigs/$c"
	cp -p $ctg /cluster/bluearc/hg17/contigs/$c
    end
    #	Check how much is there:
    #	du -hsc /cluster/bluearc/hg17/contigs
    #	2.8G    /cluster/bluearc/hg17/contigs

    # Distribute contigs to /iscratch/i
    ssh kkr1u00
    mkdir /iscratch/i/gs.18/build35/unmaskedContigs
    cd /iscratch/i/gs.18/build35/unmaskedContigs
    cp -p /cluster/bluearc/hg17/contigs/* .

    # Verify same amount made it there:
    #	du -hsc /iscratch/i/gs.18/build35/unmaskedContigs
    #	2.8G    /iscratch/i/gs.18/build35/unmaskedContigs
    #	Then send them to the other 7 Iservers
    /cluster/bin/iSync

    #	Go to the small cluster for this business:
    ssh kki

    mkdir -p /cluster/data/hg17/bed/simpleRepeat
    cd /cluster/data/hg17/bed/simpleRepeat
    mkdir trf
    cat << '_EOF_' > runTrf
#!/bin/csh -fe
#
set path1 = $1
set inputFN = $1:t
set outpath = $2
set outputFN = $2:t
mkdir -p /tmp/$outputFN
cp $path1 /tmp/$outputFN
pushd .
cd /tmp/$outputFN
/cluster/bin/i386/trfBig -trf=/cluster/bin/i386/trf $inputFN /dev/null -bedAt=$outputFN -tempDir=/tmp
popd
rm -f $outpath
cp -p /tmp/$outputFN/$outputFN $outpath
rm -fr /tmp/$outputFN/*
rmdir --ignore-fail-on-non-empty /tmp/$outputFN
'_EOF_'
    # << this line makes emacs coloring happy
    chmod +x runTrf

    cat << '_EOF_' > gsub
#LOOP
./runTrf {check in line+ $(path1)}  {check out line trf/$(root1).bed}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    ls -1S /iscratch/i/gs.18/build35/unmaskedContigs/*.fa > genome.lst
    gensub2 genome.lst single gsub jobList
    para create jobList
    para try
    para check
    para push
    para check
# Completed: 380 of 380 jobs
# CPU time in finished jobs:      13230s     220.49m     3.67h    0.15d  0.000 y
# IO & Wait Time:                  2078s      34.64m     0.58h    0.02d  0.000 y
# Average job time:                  40s       0.67m     0.01h    0.00d
# Longest job:                     1590s      26.50m     0.44h    0.02d
# Submission to last job:          2504s      41.73m     0.70h    0.03d

    liftUp simpleRepeat.bed /cluster/data/hg17/jkStuff/liftAll.lft \
	warn trf/*.bed  > lu.out 2>&1

    # Load into the database:
    ssh hgwdev
    cd /cluster/data/hg17/bed/simpleRepeat
    /cluster/bin/i386/hgLoadBed hg17 simpleRepeat simpleRepeat.bed \
      -sqlTable=$HOME/kent/src/hg/lib/simpleRepeat.sql
    #	Loaded 629076 elements of size 16
    #	Compare with previous assembly
    featureBits hg17 simpleRepeat
    #	54952425 bases of 2866216770 (1.917%) in intersection

    #	with previous randoms
    featureBits hg17 simpleRepeat
    #	54964044 bases of 3096628158 (1.775%) in intersection
    featureBits hg16 simpleRepeat
    #	54320136 bases of 2865248791 (1.896%) in intersection
    #	GAPS weren't in hg17 yet at this point, after gaps added:
    #	featureBits hg17 simpleRepeat
    #	54964044 bases of 2867328468 (1.917%) in intersection
    #	featureBits -countGaps hg17 simpleRepeat
    #	54964044 bases of 3096628158 (1.775%) in intersection

###########################################################################
# CREATE MICROSAT TRACK (done 2006-7-5 JK)
     ssh hgwdev
     cd /cluster/data/hg17/bed
     mkdir microsat
     cd microsat
     awk '($5==2 || $5==3) && $6 >= 15 && $8 == 100 && $9 == 0 {printf("%s\t%s\t%s\t%dx%s\n", $1, $2, $3, $6, $16);}' ../simpleRepeat/simpleRepeat.bed > microsat.bed 
    /cluster/bin/i386/hgLoadBed hg17 microsat microsat.bed


# PROCESS SIMPLE REPEATS INTO MASK (DONE - 2004-05-21 - Hiram)
    #	re-done with new randoms - 2004-06-03 - Hiram
    # After the simpleRepeats track has been built, make a filtered version 
    # of the trf output: keep trf's with period <= 12:
    ssh eieio
    cd /cluster/data/hg17/bed/simpleRepeat
    mkdir -p trfMask
    foreach f (trf/*.bed)
      awk '{if ($5 <= 12) print;}' $f > trfMask/$f:t
    end

    #	EXPERIMENT, at a filter of <= 12, we have coverage:
    #	20904399 bases of 2867328468 (0.729%) in intersection
    #	at a filter of <= 9, we have coverage:
    #	19271270 bases of 2867328468 (0.672%) in intersection


    # Lift up filtered trf output to chrom coords as well:
    cd /cluster/data/hg17
    mkdir bed/simpleRepeat/trfMaskChrom
    foreach c ( `cat chrom.lst` )
      if (-e $c/lift/ordered.lst) then
        perl -wpe 's@(\S+)@bed/simpleRepeat/trfMask/$1.bed@' \
          $c/lift/ordered.lst > $c/lift/oTrf.lst
        liftUp bed/simpleRepeat/trfMaskChrom/chr$c.bed \
          jkStuff/liftAll.lft warn `cat $c/lift/oTrf.lst`
      endif
      if (-e $c/lift/random.lst) then
        perl -wpe 's@(\S+)@bed/simpleRepeat/trfMask/$1.bed@' \
           $c/lift/random.lst > $c/lift/rTrf.lst
        liftUp bed/simpleRepeat/trfMaskChrom/chr${c}_random.bed \
          jkStuff/liftAll.lft warn `cat $c/lift/rTrf.lst`
      endif
    end

# MASK SEQUENCE BOTH REPEATMASKER AND SIMPLE REPEAT/TRF (DONE - 2004-05-25)
#							 -Hiram
    #	re-done with new randoms - 2004-06-03 - Hiram
    # This used to be done right after RepeatMasking.  Now, we mask with 
    # TRF as well, so do this after the "PROCESS SIMPLE REPEATS" step above,
    #	and after Repeat Masker is complete.
    ssh eieio
    cd /cluster/data/hg17

    # copied these scripts from hg16 - reset the lines that make
    # the chrom list to work on, reset the wild cards that find all the
    # contig .fa's

    # Make chr*.fa from contig .fa
    #  Copied chrFa.sh from hg16/jkStuff, renamed it to chrFa.csh
    time ./jkStuff/chrFa.csh > scriptsOutput/chrFa.out 2>&1 &
    #	real    13m18.512s
    #	user    9m1.670s
    #	sys     1m7.290s

    #- Soft-mask (lower-case) the contig and chr .fa's
    time ./jkStuff/makeFaMasked.csh > scriptsOutput/maFaMasked.out 2>&1
    #	real    29m31.623s
    #	user    13m49.700s
    #	sys     5m58.750s
    #- Make hard-masked .fa.masked files as well:
    time ./jkStuff/makeHardMasked.csh > scriptsOutput/maHardMasked.out 2>&1

    #- Create the bothMasksNib/ directory
    time ./jkStuff/makeNib.csh > scriptsOutput/maNib.out 2>&1
    #	real    14m41.694s
    #	user    6m28.000s
    #	sys     1m42.500s

    # Make symbolic links from /gbdb/hg17/nib to the real nibs.
    ssh hgwdev
    mv nib nib.raw
    mv bothMasksNib nib
    rm /gbdb/hg17/nib/*.nib
    ln -s `pwd`/nib/* /gbdb/hg17/nib

    # Load /gbdb/hg17/nib paths into database and save size info.
    hgsql hg17  < ~/kent/src/hg/lib/chromInfo.sql
    cd /cluster/data/hg17
    hgNibSeq -preMadeNib hg17 /gbdb/hg17/nib */chr*.fa
    #	3096628158 total bases

    #	Should be the same size as before
    hgsql -N -e "select chrom,size from chromInfo order by chrom" hg17 \
	> chrom.sizes.masked
    diff chrom.sizes chrom.sizes.masked
    #	should be no output at all, thus:
    rm chrom.sizes.masked

    # Copy the masked contig fa to /scratch and /iscratch
    #	And everything else we will need for blastz runs, etc ...
    #	Best to do this sequence first to /cluster/bluearc/scratch,
    #	which is going to be the source for the /scratch copy.
    #	And then from there to the /iscratch
    #	Make sure you are on the fileserver for the original source:
    ssh eieio
    mkdir -p /cluster/bluearc/scratch/hg/gs.18/build35
    cd /cluster/bluearc/scratch/hg/gs.18/build35

    #	these copies take less than 2 minutes each
    mkdir bothMaskedNibs
    cp -p /cluster/data/hg17/nib/*.nib ./bothMaskedNibs
    mkdir maskedContigs
    foreach chrom ( `cat /cluster/data/hg17/chrom.lst` )
	cp -p /cluster/data/hg17/${chrom}/N{C,G,T}_*/N{C,G,T}_??????.fa \
		./maskedContigs
	echo "done ${chrom}"
    end
    #	make sure you have them all:
    ls maskedContigs | wc -l
    #	380
    wc -l /cluster/data/hg17/contig.lst
    #	380
    mkdir rmsk
    foreach chrom ( `cat /cluster/data/hg17/chrom.lst` )
	cp -p /cluster/data/hg17/${chrom}/*.out ./rmsk
	echo "done ${chrom}"
    end

    #	Now, go to the destination for /iscratch and copy from the
    #	bluearc
    ssh kkr1u00
    mkdir -p /iscratch/i/gs.18/build35
    cd /iscratch/i/gs.18/build35
    #	This takes about 5 minutes
    rsync -arlv /cluster/bluearc/scratch/hg/gs.18/build35/ .

    time /cluster/bin/iSync
    #	real    7m27.649s

    # request rsync of /cluster/bluearc/scratch to the KiloKluster /scratch

# LOAD ctgPos table - Contig position track (DONE - 2004-06-08 - Hiram)
    #	After fixing up hgCtgPos to accept the -chromLst argument, simply:
    cd /cluster/data/hg17
    hgCtgPos -chromLst=chrom.lst hg17 .

# GOLD AND GAP TRACKS (DONE - 2004-05-21 - Hiram)
    #	RE-DONE with new randoms - 2004-06-03 - Hiram
    ssh hgwdev
    cd /cluster/data/hg17
    hgGoldGapGl -noGl -chromLst=chrom.lst hg17 /cluster/data/hg17 .
    #	Disappointing to see this create so many tables ...
    #	_gap and _gold for each chrom

    # Create the contig.gl files - XXX - NCBI doesn't deliver
    # contig_overlaps.agp - 2004-06-18 - this is beginning to come
    # together and there is now a contig_overlaps.agp file
    cd /cluster/store5/gs.18/build35
    #	combine the various psLayout attempts on different sections of
    #	clones
    ./combineContigOverlaps.sh
    #	Turn contig_overlaps.agp into gl files
    ~hiram/bin/i386/agpToGl contig_overlaps.agp . -md=seq_contig.md
    # Create chromosome gl files  (had to fix liftUp to do the NC_ properly)
    jkStuff/liftGl.csh contig.gl
    #
    #	Need to remove these PAR clone names from chrY.gl
    rm -f /tmp/chrY.gl
    sed -e "/^AL954722.18/d; /^BX537334.4/d; /^BX000483.7/d; \
	/^BX908402.3/d; /^BX649635.3/d; /^BX119919.5/d; \
	/^AC079176.15/d; /^AC097314.27/d; /^AC006209.25/d; \
	/^AJ271735.1/d; /^AJ271736.1/d" Y/chrY.gl > /tmp/chrY.gl
    rm -f Y/chrY.gl
    mv /tmp/chrY.gl Y/chrY.gl

#  After contig.gl files have been made from contig_overlaps.agp
#	The sed fixes the Celera clones that are marked phase W
#	Call that phase 3 instead,
#	Delete the Celera AACC clones, they are not in this assembly,
#	And fix the line of AC018743 to add it to the assembly, it was a
#	last minute addition by Terry that didn't get carried into the
#	NCBI sequence.inf file.  And remove the older versions of five
#	clones that got left in by mistake at NCBI
    #	AC004491.1=AC004491.2 AC004921.1=AC004921.2 AC004983.2=AC004983.3
    #	AC005088.2=AC005088.3 AC006014.2=AC006014.3 AC099654.4=AC099654.5 
#	And finally the grep selects only those things for_assembly
    cd /cluster/data/hg17
    egrep "for_assembly|AC018743" /cluster/store5/gs.18/ncbi/sequence.inf | \
    sed -e "s/\tW\t/\t3\t/; /^AACC010000.*/d; /^AC004491.1.*/d; \
	/^AC004921.1.*/d; /^AC004983.2.*/d; /^AC005088.2.*/d; \
	/^AC006014.2.*/d; /^AC099654.4.*/d; \
	s/AC018743.27\t31791062\t466818\t1\tD\tUn\t-\tBCM\tRP11-289M22\tSIZE:2big/AC018743.27\t31791062\t466818\t1\t-\t(12)\t-\tBCM\tRP11-289M22\tfor_assembly/" \
	> sequence.inf
    cd /cluster/data/hg17
    hgGoldGapGl -chromLst=chrom.lst hg17 /cluster/store5/gs.18 build35
    $HOME/bin/i386/hgClonePos -chromLst=chrom.lst hg17 \
	/cluster/data/hg17 ./sequence.inf /cluster/store5/gs.18 -maxErr=3 \
	-maxWarn=2000 2> clone.pos.errors
    #	Extract all the PAR clones for chrX from clonePos, change the X
    #	to Y, fixup the coordinates on the last three, and load this
    #	data in on the clonePos table in addition to what is there
    #	already.
    cat << '_EOF_' > chrY.par.clonePos
BX640545.2      34821   3       chrY    0       34250   F
AL954722.18     37771   3       chrY    84821   122592  F
BX537334.4      36872   3       chrY    120592  157464  F
BX000483.7      15918   3       chrY    155466  171384  F
AL954664.17     39200   3       chrY    251384  290307  F
BX000476.5      33340   3       chrY    282188  315528  F
AL732314.18     218723  3       chrY    313528  532251  F
BX004827.18     119555  3       chrY    479050  600112  F
AL683871.15     175765  3       chrY    598112  773877  F
AL672311.26     115998  3       chrY    771877  887875  F
AL672277.20     131682  3       chrY    885875  1017557 F
BX908402.3      36556   3       chrY    1067557 1104113 F
BX649635.3      43709   3       chrY    1154113 1197822 F
BX649553.5      90286   3       chrY    1347822 1438108 F
BX296563.3      21008   3       chrY    1488108 1509117 F
BX119906.16     35666   3       chrY    1507116 1542782 F
AL683870.15     162377  3       chrY    1541782 1704175 F
AL691415.17     45085   3       chrY    1702175 1747265 F
AL683807.22     189825  3       chrY    1745260 1935086 F
AL672040.10     117297  3       chrY    1933086 2050383 F
BX004859.8      63432   3       chrY    2048380 2111815 F
BX119919.5      55442   3       chrY    2261815 2317257 F
AC079176.15     186278  3       chrY    2311674 2497952 F
AC097314.27     80501   3       chrY    2495948 2576449 F
AC006209.25     141759  3       chrY    2551122 2692881 F
AJ271735.1      240000  3       chrY    57302979        57543030        F
AJ271736.1      158661  3       chrY    57543030        57701691        F
'_EOF_'

hgsql -e 'load data local infile "chrY.par.clonePos" into table clonePos;' hg17

    #	We have the following errors
# Processing /cluster/data/hg17/Y/chrY.gl
# Clone BX640545 is on chromosomes chrX and chrY.  Ignoring chrY
# Clone AL954722 is on chromosomes chrX and chrY.  Ignoring chrY
# ... etc for all the PAR clones
# ... And there are an unknown number of these:
# AB000359 is in ./sequence.inf but not in ooDir/*/*.gl
# AB000360 is in ./sequence.inf but not in ooDir/*/*.gl

#  gc5Base wiggle TRACK (DONE - 2004-05-22 - Hiram)
    #	This previously was a script that ran through each nib
    #	Recently transformed into a mini cluster run.
    #	Re-DONE with the new randoms - 2004-06-04
    ssh kki
    mkdir /cluster/data/hg17/bed/gc5Base
    cd /cluster/data/hg17/bed/gc5Base

    mkdir wigData5 dataLimits5 wigData5_1K dataLimits5_1K

    cat << '_EOF_' > kkRun.sh
#!/bin/sh
NIB=$1

chr=${NIB/.nib/}
chrom=${chr#chr}

hgGcPercent -chr=${chr} -doGaps -file=stdout -win=5 hg17 \
        /iscratch/i/gs.18/build35/bothMaskedNibs | \
    grep -w GC | \
    awk '{if (($3-$2) >= 5) {printf "%d\t%.1f\n", $2+1, $5/10.0} }' | \
    wigAsciiToBinary -dataSpan=5 -chrom=${chr} \
        -wibFile=wigData5/gc5Base_${chrom} \
            -name=${chrom} stdin 2> dataLimits5/${chr}
'_EOF_'
    # << this line makes emacs coloring happy
    chmod +x kkRun.sh

    ls /iscratch/i/gs.18/build35/bothMaskedNibs > nibList
    cat << '_EOF_' > gsub
#LOOP
./kkRun.sh $(path1)
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    gensub2 nibList single gsub jobList
    para create jobList
    para try, check, ... etc
# Completed: 46 of 46 jobs
# CPU time in finished jobs:       5251s      87.51m     1.46h    0.06d  0.000 y
# IO & Wait Time:                   130s       2.17m     0.04h    0.00d  0.000 y
# Average job time:                 117s       1.95m     0.03h    0.00d
# Longest job:                      413s       6.88m     0.11h    0.00d
# Submission to last job:           475s       7.92m     0.13h    0.01d

    # load the .wig files back on hgwdev:
    ssh hgwdev
    cd /cluster/data/hg17/bed/gc5Base
    hgLoadWiggle -pathPrefix=/gbdb/hg17/wib/gc5Base hg17 gc5Base wigData5/*.wig
    # and symlink the .wib files into /gbdb
    mkdir /gbdb/hg17/wib/gc5Base
    ln -s `pwd`/wigData5/*.wib /gbdb/hg17/wib/gc5Base

    #	And then the zoomed data view
    ssh kki
    cd /cluster/data/hg17/bed/gc5Base
    mkdir wigData5_1K dataLimits5_1K

    cat << '_EOF_' > kkRunZoom.sh
#!/bin/sh
NIB=$1

chr=${NIB/.nib/}
chrom=${chr#chr}

hgGcPercent -chr=${chr} -doGaps -file=stdout -win=5 hg17 \
        /iscratch/i/gs.18/build35/bothMaskedNibs | \
    grep -w GC | \
    awk '{if (($3-$2) >= 5) {printf "%d\t%.1f\n", $2+1, $5/10.0} }' | \
    wigZoom -dataSpan=1000 stdin | wigAsciiToBinary -dataSpan=1000 \
	-chrom=${chr} -wibFile=wigData5_1K/gc5Base_${chrom}_1K \
            -name=${chrom} stdin 2> dataLimits5_1K/${chr}
'_EOF_'
    # << this line makes emacs coloring happy
    chmod +x kkRunZoom.sh

    cat << '_EOF_' > gsubZoom
#LOOP
./kkRunZoom.sh $(path1)
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    gensub2 nibList single gsubZoom jobListZoom
    para create jobListZoom
    para try ... check ... etc ...
# Completed: 46 of 46 jobs
# CPU time in finished jobs:       5216s      86.93m     1.45h    0.06d  0.000 y
# IO & Wait Time:                    34s       0.57m     0.01h    0.00d  0.000 y
# Average job time:                 114s       1.90m     0.03h    0.00d
# Longest job:                      415s       6.92m     0.12h    0.00d
# Submission to last job:           469s       7.82m     0.13h    0.01d

    #	Then load these .wig files into the same database as above
    ssh hgwdev
    hgLoadWiggle -pathPrefix=/gbdb/hg17/wib/gc5Base \
	-oldTable hg17 gc5Base wigData5_1K/*.wig
    # and symlink these .wib files into /gbdb
    mkdir -p /gbdb/hg17/wib/gc5Base
    ln -s `pwd`/wigData5_1K/*.wib /gbdb/hg17/wib/gc5Base

# AUTO UPDATE GENBANK MRNA RUN  (DONE - 2004-06-08 - Hiram)
    ssh eieio
    cd /cluster/data/genbank
    # This is a new organism, edit the etc/genbank.conf file and add:
	# hg17
	hg17.genome = /scratch/hg/gs.18/build35/bothMaskedNibs/chr*.nib
	hg17.lift = /cluster/store5/gs.18/build35/jkStuff/liftAll.lft
	hg17.genbank.est.xeno.load = yes
	hg17.mgcTables.default = full
	hg17.mgcTables.mgc = all
	hg17.downloadDir = hg17

    #	Do the refseq's first, they are the quick ones
    ssh eieio
    cd /cluster/data/genbank
    nice bin/gbAlignStep -srcDb=refseq -type=mrna -verbose=1 -initial hg17
    #	 logFile: var/build/logs/2004.05.25-13:41:07.hg17.initalign.log
    #	checking that log, or watching the batch on kk, you can find
    #	where the batch is running and after it is done get the time:
    cd /cluster/store6/genbank/work/initial.hg17/align
    para time > time
    cat time
# Completed: 9500 of 9500 jobs
# CPU time in finished jobs:      62241s    1037.35m    17.29h    0.72d  0.002 y
# IO & Wait Time:                 33719s     561.98m     9.37h    0.39d  0.001 y
# Average job time:                  10s       0.17m     0.00h    0.00d
# Longest job:                     1062s      17.70m     0.29h    0.01d
# Submission to last job:          1063s      17.72m     0.30h    0.01d

    # Load the results from the above
    ssh hgwdev
    cd /cluster/data/genbank
    nice bin/gbDbLoadStep -verbose=1 -drop -initialLoad hg17

#	To get the genbank started, the above results need to be
#	moved out of the way.  These things can be removed if there are
#	no problems to debug
    ssh eieio
    cd /cluster/data/genbank/work
    mv initial.hg17 initial.hg17.refseq.mrna

    cd /cluster/data/genbank
    nice bin/gbAlignStep -srcDb=genbank -type=mrna -verbose=1 -initial hg17
    #	logFile: var/build/logs/2004.06.04-10:47:21.hg17.initalign.log
    #	One job was hung up, after killing it on its node, the batch
    #	finished in a few minutes.
# Completed: 35720 of 35720 jobs
# CPU time in finished jobs:    5161424s   86023.74m  1433.73h   59.74d  0.164 y
# IO & Wait Time:                144149s    2402.48m    40.04h    1.67d  0.005 y
# Average job time:                 149s       2.48m     0.04h    0.00d
# Longest job:                    18306s     305.10m     5.08h    0.21d
# Submission to last job:         35061s     584.35m     9.74h    0.41d

    ssh hgwdev
    cd /cluster/data/genbank
    #	some kind of error happened here, had to remove a lock file to
    #	get this to proceed  (this same thing happened again the second
    #	time around)
    nice bin/gbDbLoadStep -verbose=1 -drop -initialLoad hg17

    ssh eieio
    cd /cluster/data/genbank/work
    mv initial.hg17 initial.hg17.genbank.mrna
    cd /cluster/data/genbank
    nice bin/gbAlignStep -srcDb=genbank -type=est -verbose=1 -initial hg17
# Completed: 189240 of 189240 jobs
# CPU time in finished jobs:   97172120s 1619535.33m 26992.26h 1124.68d  3.081 y
# IO & Wait Time:               1507789s   25129.82m   418.83h   17.45d  0.048 y
# Average job time:                 521s       8.69m     0.14h    0.01d
# Longest job:                    33165s     552.75m     9.21h    0.38d
# Submission to last job:        126988s    2116.47m    35.27h    1.47d

    ssh hgwdev
    cd /cluster/data/genbank
    time nice bin/gbDbLoadStep -verbose=1 -drop -initialLoad hg17
    #	real    440m42.750s
    #	user    69m7.810s
    #	sys     23m18.640s
    #	This is ~7.5 hours

    #	If the above is all OK, ask Mark to put this assembly on
    #	the daily updates.

# CPGISLANDS (DONE - 2004-05-25 - Hiram)
    #	Re-DONE with new randoms - 2004-06-04 - Hiram
    ssh hgwdev
    mkdir -p /cluster/data/hg17/bed/cpgIsland
    cd /cluster/data/hg17/bed/cpgIsland

    # Build software from Asif Chinwalla (achinwal@watson.wustl.edu)
    cvs co hg3rdParty/cpgIslands
    cd hg3rdParty/cpgIslands
    make
    #	gcc readseq.c cpg_lh.c -o cpglh.exe
    mv cpglh.exe /cluster/data/hg17/bed/cpgIsland/
    
    # cpglh.exe requires hard-masked (N) .fa's.  
    # There may be warnings about "bad character" for IUPAC ambiguous 
    # characters like R, S, etc.  Ignore the warnings.  
    ssh eieio
    cd /cluster/data/hg17/bed/cpgIsland
    foreach f (../../*/chr*.fa.masked)
      set fout=$f:t:r:r.cpg
      echo running cpglh on $f to $fout
      ./cpglh.exe $f > $fout
    end
    #	the warnings:
    # Bad char 0x52 = 'R' at line 2046, base 102229, sequence chr16_random
    # Bad char 0x4d = 'M' at line 1216113, base 60805573, sequence chr3
    # Bad char 0x52 = 'R' at line 1216118, base 60805801, sequence chr3
    # Bad char 0x52 = 'R' at line 1216118, base 60805801, sequence chr3
    #	real    21m47.823s
    #	user    18m30.810s
    #	sys     1m13.420s

    # Transform cpglh output to bed +
    cat << '_EOF_' > filter.awk
/* Input columns: */
/* chrom, start, end, len, CpG: cpgNum, perGc, cpg:gpc, observed:expected */
/* chr1\t 41776\t 42129\t 259\t CpG: 34\t 65.8\t 0.92\t 0.94 */
/* Output columns: */
/* chrom, start, end, name, length, cpgNum, gcNum, perCpg, perGc, obsExp */
/* chr1\t41775\t42129\tCpG: 34\t354\t34\t233\t19.2\t65.8\to0.94 */
{
$2 = $2 - 1;
width = $3 - $2;
printf("%s\t%d\t%s\t%s %s\t%s\t%s\t%0.0f\t%0.1f\t%s\t%s\n",
       $1, $2, $3, $5,$6, width,
       $6, width*$7*0.01, 100.0*2*$6/width, $7, $9);
}
'_EOF_'
    # << this line makes emacs coloring happy
    awk -f filter.awk chr*.cpg > cpgIsland.bed

    ssh hgwdev
    cd /cluster/data/hg17/bed/cpgIsland
    hgLoadBed hg17 cpgIslandExt -tab -noBin \
      -sqlTable=$HOME/kent/src/hg/lib/cpgIslandExt.sql cpgIsland.bed
    #	Reading cpgIsland.bed
    #	Loaded 27801 elements of size 10
    #	Sorted
    #	Saving bed.tab
    #	Loading hg17

# MAKE HGCENTRALTEST BLATSERVERS ENTRY (DONE - 2004-05-25 - Heather)
    ssh hgwdev
    hgsql -e 'INSERT INTO blatServers (db, host, port, isTrans) \
	VALUES("hg17", "blat12", "17778", "1"); \
	INSERT INTo blatServers (db, host, port, isTrans) \
	VALUES("hg17", "blat12", "17779", "0");' \
	-h genome-testdb hgcentraltest

# PREPARE CLUSTER FOR BLASTZ RUNS (DONE - 2004-05-26 - Hiram)
    #	Re-DONE with new randoms - 2004-06-03 - Hiram

    ssh eieio
    mkdir /cluster/bluearc/scratch/hg/gs.18/build35/rmsk.spec
    cd /cluster/bluearc/scratch/hg/gs.18/build35/rmsk.spec
    ln -s ../rmsk/*.out .
    #	This takes 40 minutes run as a script, to hurry it up it has
    #	been converted to a mini cluster run
    cat << '_EOF_' > runArian.sh
#!/bin/sh
for FN in *.out
do
    echo /cluster/bluearc/RepeatMasker030619/DateRepsinRMoutput.pl \
	${FN} -query human -comp rat -comp mouse
done
'_EOF_'
    # << emacs
    chmod +x runArian.sh
    ssh kki
    cd /cluster/bluearc/scratch/hg/gs.18/build35/rmsk.spec
    ./runArian.sh > jobList
    para create jobList
    para try, ... check ... push ... etc ...
# Completed: 46 of 46 jobs
# CPU time in finished jobs:        668s      11.14m     0.19h    0.01d  0.000 y
# IO & Wait Time:                   514s       8.56m     0.14h    0.01d  0.000 y
# Average job time:                  26s       0.43m     0.01h    0.00d
# Longest job:                       86s       1.43m     0.02h    0.00d
# Submission to last job:           108s       1.80m     0.03h    0.00d

    #	Now extract each one, 1 = Rat, 2 = Mouse
    ssh eieio
    cd /cluster/bluearc/scratch/hg/gs.18/build35

    mkdir linSpecRep.notInRat linSpecRep.notInMouse
    foreach f (rmsk.spec/*.out_rat_mus)
        set base = $f:t:r:r
        echo "$f -> $base.out.spec"
        /cluster/bin/scripts/extractLinSpecReps 1 $f > \
                        linSpecRep.notInRat/$base.out.spec
        /cluster/bin/scripts/extractLinSpecReps 2 $f > \
                        linSpecRep.notInMouse/$base.out.spec
    end
    #	There is actually no difference at all between these two results.
    #	copy to iscratch
    ssh kkr1u00
    cd /iscratch/i/gs.18/build35
    rsync -arlv /cluster/bluearc/scratch/hg/gs.18/build35/ .
    /cluster/bin/iSync
    # request rsync of /cluster/bluearc/scratch to the KiloKluster /scratch

# COPY DATA TO GOLDEN PATH LOCATIONS (DONE - 2004-06-04 - Hiram)
    ssh hgwdev
    mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/chromosomes
    cd /cluster/data/hg17
    #	Beware, this backgrounding of the gzips can be hard on hgwdev.
    #	You could wait until after the copy then run one gzip to do them all
    foreach chrom ( `cat chrom.lst` )
	cp -p ${chrom}/*.fa /usr/local/apache/htdocs/goldenPath/hg17/chromosomes
	gzip \
	/usr/local/apache/htdocs/goldenPath/hg17/chromosomes/chr${chrom}*.fa &
	echo "done ${chrom}"
    end
    cd /usr/local/apache/htdocs/goldenPath/hg17/chromosomes
    gzip *.fa


# FOSMID END PAIRS TRACK (2004-06-09 kate)
# Corrected upper size limit to 50Kbp, reran pslPairs, 
# and reloaded (2004-07-15 kate)

    # Use latest fosmid ends data prepared by Terry Furey.
    # He says there is no on-going work on fosmid ends, so this
    # should suffice indefinitely ?  Move/link this stuff into
    # central data area.
    ssh eieio
    cd /cluster/data/ncbi
    mkdir -p fosends/human
    ln -s /cluster/store1/fosends.3 fosends/human
    cd fosends/human/fosends.3
    faSize fosEnds.fa
       # 579735181 bases (369769 N's 579365412 real) in 1087670 sequences 
       # 580M bases in 1M sequences
    # create link in /gbdb/ncbi/fosends/human ?

    # use pre-split fosend files, and associated list for cluster run
    # Sequences are in /cluster/bluearc/hg/fosEnds
    cp /cluster/bluearc/booch/fosends/fosEnds.lst /cluster/bluearc/hg/fosEnds
    
    # run on rack9 since kilokluster is busy
    ssh kk9
    cd /cluster/data/hg17
    mkdir -p bed/fosends
    cd bed/fosends
    mkdir -p run
    cd run
    ls -1S /scratch/hg/gs.18/build35/maskedContigs/*.fa \
                > contigs.lst
    cp /cluster/bluearc/hg/fosEnds/fosEnds.lst fosEnds.lst
        # 380 contigs vs 97 fosEnd files -> 40K jobs
    # send output to kksilo, as it can better handle the NFS load
    mkdir -p /cluster/store7/kate/hg17/fosends/out
    ln -s /cluster/store7/kate/hg17/fosends/out ../out
cat > gsub << 'EOF'
#LOOP
/cluster/bin/i386/blat $(path1) $(path2) -ooc=/scratch/hg/h/11.ooc {check out line+ /cluster/data/hg17/bed/fosends/out/$(root2)/$(root1).$(root2).psl}
#ENDLOOP
'EOF'
    # << emacs
    gensub2 contigs.lst fosEnds.lst gsub jobList
    foreach f (`cat fosEnds.lst`)
        set d = $f:r:t
        echo $d
        mkdir -p /cluster/data/hg17/bed/fosends/out/$d
    end

    para create jobList
        # 36860 jobs
    para try
    para check
    para push
# CPU time in finished jobs:    1655943s   27599.05m   459.98h   19.17d  0.053 y
# IO & Wait Time:                101145s    1685.75m    28.10h    1.17d  0.003 y
# Average job time:                  48s       0.79m     0.01h    0.00d
# Longest job:                     1294s      21.57m     0.36h    0.01d
# Submission to last job:         19269s     321.15m     5.35h    0.22d

    # sort, filter, and lift alignments
    ssh eieio
    cd /cluster/data/hg17/bed/fosends
    pslSort dirs raw.psl temp out/fosEnds*
    pslReps  -nearTop=0.01 -minCover=0.70 -minAli=0.85 -noIntrons raw.psl \
                        fosEnds.psl /dev/null
        # Processed 84096767 alignments

    # cleanup
    rm -r temp
    rm raw.psl
    rm -fr out /cluster/store7/kate/hg17/fosends

    mkdir lifted
    liftUp lifted/fosEnds.lifted.psl \
                /cluster/data/hg17/jkStuff/liftAll.lft warn fosEnds.psl
    pslSort dirs fosEnds.sorted.psl temp lifted
    rmdir temp
    wc -l *.sorted.psl
        # 1693693 fosEnds.sorted.psl
 
    set ncbiDir = /cluster/data/ncbi/fosends/human/fosends.3
    ~/bin/i386/pslPairs -tInsert=5000 -minId=0.94 -noBin -min=30000 -max=50000 -slop -short -long -orphan -mismatch -verbose fosEnds.sorted.psl $ncbiDir/fosEnds.pairs all_fosends fosEnds

    # create header required by "rdb" tools
    # TODO: replace w/ awk & sort
    echo 'chr\tstart\tend\tclone\tscore\tstrand\tall\tfeatures\tstarts\tsizes' > header
    echo '10\t10N\t10N\t10\t10N\t10\t10\t10N\t10\t10' >> header
    cat header fosEnds.pairs | row score ge 300 | sorttbl chr start | headchg -del > fosEndPairs.bed
    cat header fosEnds.slop fosEnds.short fosEnds.long fosEnds.mismatch \
         fosEnds.orphan \
    | row score ge 300 | sorttbl chr start | headchg -del > fosEndPairsBad.bed

    extractPslLoad -noBin fosEnds.sorted.psl fosEndPairs.bed \
                fosEndPairsBad.bed | \
                        sorttbl tname tstart | headchg -del > fosEnds.load.psl

    # load into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/fosends
    hgLoadBed hg17 fosEndPairs fosEndPairs.bed \
                 -sqlTable=/cluster/home/kate/kent/src/hg/lib/fosEndPairs.sql 
        # Loaded 384558 elements

    # note - this track isn't pushed to RR, just used for assembly QA
    hgLoadBed hg17 fosEndPairsBad fosEndPairsBad.bed \
                 -sqlTable=/cluster/home/kate/kent/src/hg/lib/fosEndPairsBad.sql
        # Loaded  30830 elements

    #hgLoadPsl hg17 -nobin -table=all_fosends fosEnds.load.psl
    # NOTE: truncates file to 0 if -nobin is used
    hgLoadPsl hg17 -table=all_fosends fosEnds.load.psl
        # load of all_fosends did not go as planned: 1526991 record(s), 0 row(s) skipped, 156 warning(s) loading psl.tab

    # load sequences
    mkdir -p /gbdb/hg17/fosends
    ln -s /cluster/data/ncbi/fosends/human/fosends.3/fosEnds.fa \
                                /gbdb/hg17/fosends/fosEnds.fa
    hgLoadSeq hg17 /gbdb/hg17/fosends/fosEnds.fa
        # 1087670 sequences
       # NOTE: extFile ID is 832625 (shouldn't be so large ??) 
       # may want to reset this.


# BAC END PAIRS TRACK (DONE - 2004-06-09 kate)
    # Re-ran pslPairs with updated pairs file (2004-10-04 booch)

    # Use latest BAC ends data from NCBI
    # Checked  ftp.ncbi.nih.gov/genomes/BACENDS/homo_sapiens,
    #  and files were unchanged from Terry's last download
    #  (to /cluster/store1/bacends.4)
    # Link this stuff into central data area.
    ssh eieio
    cd /cluster/data/ncbi
    mkdir -p bacends/human
    ln -s /cluster/store1/bacends.4 bacends/human
    cd bacends/human/bacends.4
    faSize BACends.fa
        # 400230494 bases (2743171 N's 397487323 real) in 832614 sequences
        # 400M bases in 800K sequences

    # use pre-split bacends files, and associated list for cluster run
    ssh kk
    cd /cluster/data/hg17
    mkdir -p bed/bacends
    cd bed/bacends
    mkdir run
    cd run
    ls -1S /scratch/hg/gs.18/build35/maskedContigs/*.fa > contigs.lst
    ls -1S /cluster/bluearc/hg/bacEnds/hs/*.fa > bacends.lst
        # 380 contigs vs 98 bacends files -> 40K jobs

    # send output to kksilo, as it can better handle the NFS load
    # (these are quick jobs)
    mkdir -p /cluster/store7/kate/hg17/bacends/out
    ln -s /cluster/store7/kate/hg17/bacends/out ../out
cat > gsub << 'EOF'
#LOOP
/cluster/bin/i386/blat $(path1) $(path2) -ooc=/scratch/hg/h/11.ooc {check out line+ /cluster/data/hg17/bed/bacends/out/$(root2)/$(root1).$(root2).psl}
#ENDLOOP
'EOF'
# << emacs
    gensub2 contigs.lst bacends.lst gsub jobList
    foreach f (`cat bacends.lst`)
        set d = $f:r:t
        echo $d
        mkdir -p /cluster/data/hg17/bed/bacends/out/$d
    end

    para create jobList
        # 37240 jobs written to batch
    para try
    para check
    para push
# CPU time in finished jobs:    1573932s   26232.19m   437.20h   18.22d  0.050 y
# IO & Wait Time:                122751s    2045.86m    34.10h    1.42d  0.004 y
# Average job time:                  46s       0.76m     0.01h    0.00d
# Longest job:                     3312s      55.20m     0.92h    0.04d
# Submission to last job:          7148s     119.13m     1.99h    0.08d

    cd ../out/BACends000
    pslCheck *.psl
#Error: invalid PSL: AZ519021:1-575 NT_004559:1306426-1608347 - NT_004559.BACends000.psl:1101
#AZ519021 query block 3 start 283 < previous block end 575
    # NOTE: inquired with JK regarding these results

    # lift alignments
    ssh eieio
    cd /cluster/data/hg17/bed/bacends
    pslSort dirs raw.psl temp out/BACends*
    # takes hours ?

        # 37240 files in 98 dirs
        # Got 37240 files 193 files per mid file
    pslReps -nearTop=0.02 -minCover=0.60 -minAli=0.85 -noIntrons \
                raw.psl  bacEnds.psl /dev/null
        # Processed 52291246 alignments
    mkdir lifted
    liftUp lifted/bacEnds.lifted.psl \
                /cluster/data/hg17/jkStuff/liftAll.lft warn bacEnds.psl
    pslSort dirs bacEnds.sorted.psl temp lifted

    # cleanup
    rmdir temp
    rm -fr out /cluster/store7/kate/hg17/bacends

    wc -l *.sorted.psl
        # 2497227 bacEnds.sorted.psl

    set ncbiDir = /cluster/data/ncbi/bacends/human/bacends.4
    ~/bin/i386/pslPairs -tInsert=10000 -minId=0.91 -noBin -min=25000 -max=350000 -slopval=10000 -hardMax=500000 -slop -short -long -orphan -mismatch -verbose bacEnds.sorted.psl $ncbiDir/bacEndPairs.txt all_bacends bacEnds

    # create header required by "rdb" tools
    # TODO: replace w/ awk & sort
    echo 'chr\tstart\tend\tclone\tscore\tstrand\tall\tfeatures\tstarts\tsizes' > header
    echo '10\t10N\t10N\t10\t10N\t10\t10\t10N\t10\t10' >> header
    cat header bacEnds.pairs | row score ge 300 | sorttbl chr start | headchg -del > bacEndPairs.bed
    cat header  bacEnds.slop bacEnds.short bacEnds.long bacEnds.mismatch bacEnds.orphan \
        | row score ge 300 | sorttbl chr start | headchg -del > bacEndPairsBad.bed

    extractPslLoad -noBin bacEnds.sorted.psl bacEndPairs.bed \
                bacEndPairsBad.bed | \
                        sorttbl tname tstart | headchg -del > bacEnds.load.psl

    # load into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/bacends
    hgLoadBed hg17 bacEndPairs bacEndPairs.bed \
                 -sqlTable=/cluster/home/kate/kent/src/hg/lib/bacEndPairs.sql 
        # Loaded 201380  
    # note - this track isn't pushed to RR, just used for assembly QA
    hgLoadBed hg17 bacEndPairsBad bacEndPairsBad.bed \
                 -sqlTable=/cluster/home/kate/kent/src/hg/lib/bacEndPairsBad.sql
        # Loaded 81773
    #hgLoadPsl hg17 -nobin -table=all_bacends bacEnds.load.psl
    # NOTE: truncates file to 0 if -nobin is used
    hgLoadPsl hg17 -table=all_bacends bacEnds.load.psl
        #load of all_bacends did not go as planned: 441072 record(s), 0 row(s) skipped, 30 warning(s) loading psl.tab
    # Reloaded table, 2004-07-21, and got more rows:
    # load of all_bacends did not go as planned: 1698790 record(s), 
    # 0 row(s) skipped, 63 warning(s) loading psl.tab

    # load BAC end sequences

    mkdir -p /gbdb/hg17/bacends
    ln -s /cluster/data/ncbi/bacends/human/bacends.4/BACends.fa \
                                /gbdb/hg17/bacends/BACends.fa
    hgLoadSeq hg17 /gbdb/hg17/bacends/BACends.fa
        # 158588 sequences

    # Re-ran pslPairs with updated pairs file to take advantage of new 
    # feature of # allowing comma separated lists of end accessions for each
    #  end for a clone
    
    # First, create new pairs file (bacEndPairs.txt, bacEndSingles.txt)
    mkdir /cluster/data/ncbi/bacends/human/bacends.5
    cd /cluster/data/ncbi/bacends/human/bacends.5
    cp ../bacends.4/cl_ac_gi_len .
    /cluster/bin/scripts/convertEndPairInfo cl_ac_gi_len

    # Next, re-create the bed file
    mkdir /cluster/data/hg17/bed/bacends.update
    cd /cluster/data/hg17/bed/bacends.update
    ln -s /cluster/data/hg17/bed/bacends/bacEnds.sorted.psl ./bacEnds.sorted.psl
    set ncbiDir = /cluster/data/ncbi/bacends/human/bacends.5
    ~/bin/i386/pslPairs -tInsert=10000 -minId=0.91 -noBin -min=25000 -max=350000 -slopval=10000 -hardMax=500000 -slop -short -long -orphan -mismatch -verbose bacEnds.sorted.psl $ncbiDir/bacEndPairs.txt all_bacends bacEnds

    # create header required by "rdb" tools
    # TODO: replace w/ awk & sort
    echo 'chr\tstart\tend\tclone\tscore\tstrand\tall\tfeatures\tstarts\tsizes' > header
    echo '10\t10N\t10N\t10\t10N\t10\t10\t10N\t10\t10' >> header
    cat header bacEnds.pairs | row score ge 300 | sorttbl chr start | headchg -del > bacEndPairs.bed
    cat header  bacEnds.slop bacEnds.short bacEnds.long bacEnds.mismatch bacEnds.orphan \
        | row score ge 300 | sorttbl chr start | headchg -del > bacEndPairsBad.bed

    # wc *.bed
    # 204884 2253724 20612402 bacEndPairs.bed
    # 79401  873411 6527559 bacEndPairsBad.bed

    # previous
    # wc ../bacends/*.bed
    # 201380 2215180 20280578 ../bacends/bacEndPairs.bed
    # 81773  899503 6712402 ../bacends/bacEndPairsBad.bed

    extractPslLoad -noBin bacEnds.sorted.psl bacEndPairs.bed \
                bacEndPairsBad.bed | \
                        sorttbl tname tstart | headchg -del > bacEnds.load.psl

    # load into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/bacends.update
    hgLoadBed hg17 bacEndPairs bacEndPairs.bed \
                 -sqlTable=/cluster/home/kate/kent/src/hg/lib/bacEndPairs.sql 
        # Loaded 204884  
    # note - this track isn't pushed to RR, just used for assembly QA
    hgLoadBed hg17 bacEndPairsBad bacEndPairsBad.bed \
                 -sqlTable=/cluster/home/kate/kent/src/hg/lib/bacEndPairsBad.sql
        # Loaded 79401
    #hgLoadPsl hg17 -nobin -table=all_bacends bacEnds.load.psl
    # NOTE: truncates file to 0 if -nobin is used
    hgLoadPsl hg17 -table=all_bacends bacEnds.load.psl
    # load of all_bacends did not go as planned: 1729146 record(s), 0 row(s) skipped, 70 warning(s) loading psl.tab


# PLACE ASSEMBLY CLONES - misc instructions, only somewhat relevant
#	See PLACE ASSEMBLY CLONES ON CONTIGS AND SEQUENCE below

######	 A second attempt at clone alignment###
    #	Split the clones into 3K pieces into about 1000 fa files

    #	Example:
zcat Z99916.1.fa.gz Z99774.1.fa.gz Z99756.7.fa.gz | faSplit size stdin 3000 /tmp/name.fa -lift=/tmp/name.lft -oneFile

    #	Trying this idea in unPlacedBatch
    ssh kk0
    mkdir /cluster/data/hg17/bed/contig_overlaps/unPlacedBatch
    cd /cluster/data/hg17/bed/contig_overlaps/unPlacedBatch
    ls -1S /scratch/hg/gs.18/build35/bothMaskedNibs > nibList
    ls -1S /cluster/data/hg17/bed/contig_overlaps/blatClones > cloneList
cat << '_EOF_' > gsub
#LOOP
/cluster/bin/i386/blat -fastMap -ooc=/scratch/hg/h/11.ooc -q=dna -t=dna {check in exists /scratch/hg/gs.18/build35/bothMaskedNibs/$(path1)} {check in exists+ /cluster/data/hg17/bed/contig_overlaps/blatClones/$(path2)} {check out line+ psl/$(root1)/$(root2).psl}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    mkdir psl
    cat nibList | sed -e "s/.nib//" | while read D
do
mkdir psl/$D
done

    gensub2 nibList cloneList gsub jobList
    para create jobList

# PLACE ASSEMBLY CLONES ON CONTIGS AND SEQUENCE (DONE - 2004-07-12 - Hiram)
    ssh eieio
    mkdir /cluster/data/hg17/bed/contig_overlaps
    cd /cluster/data/hg17/bed/contig_overlaps
    #	find all the clones that were used in the assembly
    sed -e "/^#.*/d" /cluster/data/hg17/ncbi_build35.agp | \
        awk '{if (!match($5,"N")) {print $6}}' | \
        sort -u > placed_in_assembly.list
    wc -l placed_in_assembly.list
    #	26872 placed_in_assembly.list
    #	These may be available from the phases files at:
    #	ftp://ftp.ncbi.nih.gov/genbank/genomes/H_sapiens
    #	Which are easily fetched with wget.  However I took a look
    #	at those and could not find all the clones in them.  There may
    #	be a versioning problem because these phases files are often
    #	updated.
    #	Fetch them from Genbank with the following three PERL scripts:
    #	[hiram@hgwdev /cluster/data/hg17/bed/contig_overlaps] ls -og *.pl
    #	-rwxrwxr-x    1     3047 May 24 18:43 bioPerlFetch.pl
    #	-rwxrwxr-x    1     2370 Jun  4 15:21 fetchGenbank.pl
    #	-rwxrwxr-x    1      700 May 24 21:47 foldEm.pl

    #	Which takes about 4 days ...
    #	Example, 
    cat << '_EOF_' > terrys.list
AC011841.7
AC018692.9
AC018743.27
AC037482.14
AL163540.11
'_EOF_'
    # << this line makes emacs coloring happy
    #	only works on hgwdev
    ssh hgwdev
    cd /cluster/data/hg17/bed/contig_overlaps
    mkdir fasta
    time ./fetchGenbank.pl terrys.list > fetchResult.out 2>&1

    #	There is a bit of behind the scenes hocus pocus going on here.
    #	This is a tedious task of comparing various lists with each
    #	other and making sure everything matches.  Manual fixups are
    #	done for the newly named 6_hla_hap* items, copies of the PAR
    #	business were duplicated so that X and Y both have the same set
    #	of clones for that.  The end result should be a directory hierarchy
    #	here with a directory for each chrom, each random, the 6_hla_hap?
    #	items and each directory contains the clones that belong to that
    #	chromosome.  The leftovers are the unplaced clones which end up
    #	in the directory called: unPlaced.  The instructions here are
    #	merely a guideline of possibilities.  Care should be taken to
    #	make sure all listings are correct and everything gets in the
    #	right place.
    ssh eieio
    #	And then make a list of all clones considered for assembly:
    sed -e "/^#.*/d" /cluster/store5/gs.18/ncbi/sequence.inf | \
	grep for_assembly | awk '{print $1}' | sort -u > sequence.list
    wc -l sequence.list
    #	46733 sequence.list
    #	Verify overlaps are correct:
    comm -12 placed_in_assembly.list sequence.list > inBoth
    comm -23 placed_in_assembly.list sequence.list > inAssemblyNotSequence
    comm -13 placed_in_assembly.list sequence.list > inSequenceNotAssembly
    wc in*
    #	    1       1      12 inAssemblyNotSequence
    #	26871   26871  301709 inBoth
    #	19862   19862  219050 inSequenceNotAssembly
    #	46734   46734  520771 total
    #	This stray one is from Terry's five additions in the final fixup
    #	phase with Greg:
    cat inAssemblyNotSequence
    #	AC018743.27
    #	Terry added: AC011841.7 AC018692.9 AC018743.27 AC037482.14 AL163540.11
    #
    #	Generate a listing that relates clones to their contigs
    sed -e "/^#.*/d" /cluster/store5/gs.18/build35/ncbi_build35.agp | \
	./contigAcc.pl > disburseEm.list
    #
    #	Using that list, sort the downloaded clones into their
    #	respective chrom directories:
    ./disburse.sh

    #	Check the number of sequences obtained:
    find ./? ./?? ./*_random ./6_hla* -type f | wc -l
    #	26872
    #	So, why is this number one more than the inBoth list ?
    #	Because, the official NCBI sequence.inf file is missing one of
    #	the clones that Terry added: AC018743.27
    #	And it shows up in our check list above as inAssemblyNotSequence
    #	It isn't exactly missing, it just isn't marked "for_assembly"

    #	OK, with everything in place, we are ready to try and find
    #	all these items in the assembly.  To run a Kluster job on one of
    #	the chroms, matching the items that are supposed to be included
    #	in that chrom.  We need to get things set up on the Iservers,
    #	psLayout is heavy into disk I/O and it brings everything down if
    #	allowed to work on any NFS filesystems for input.

    #	It appears that psLayout wants an ooc file of tile size 10
    #	I tried making one for the whole assembly but it seemed to
    #	include too much for some contigs and it caused a lot of
    #	alignments to be missed.  Thus, create an ooc file for each
    #	contig

    ssh eieio
    mkdir /cluster/bluearc/scratch/hg/gs.18/build35/contigOoc10
    cd /cluster/bluearc/scratch/hg/gs.18/build35/contigOoc10
    ls ../maskedContigs | sed -e "s/.fa//" | while read CONTIG
    do
	blat -repMatch=256 -makeOoc=${CONTIG}.10.ooc -tileSize=10 \
	    ../maskedContigs/${CONTIG}.fa \
	    ../maskedContigs/${CONTIG}.fa /dev/null
	echo "done: ${CONTIG}"
    done

    #	Copy that result to the Iservers:
    ssh kkr1u00
    mkdir /iscratch/i/gs.18/build35/contigOoc10
    cd /iscratch/i/gs.18/build35/contigOoc10
    rsync -arlv /cluster/bluearc/scratch/hg/gs.18/build35/contigOoc10/ .
    #	And, copy the clone sequences:
    mkdir /iscratch/i/gs.18/build35/clones
    cd /cluster/store5/gs.18/build35/bed/contig_overlaps
    for D in ? ?? *_random 6_hla_hap?
    do
	rsync -arlv `pwd`/${D} /iscratch/i/gs.18/build35/clones
    done
    
    /cluster/bin/iSync

    ssh kk
    cd /cluster/data/hg17/bed/contig_overlaps
    mkdir psl
    cat << '_EOF_' > runPsLayout.sh
#!/bin/sh
#       kkiPsLayout.sh <chrom> <clone> <contig>
#       where <chrom> is the chrom this contig is on
#       <clone> is one of the .fa.gz files in
#               /cluster/data/hg17/bed/contig_overlaps/*/<clone>.fa.gz
#               without the .fa.gz extension
#               This stuff has been mirrored to:
#               /iscratch/i/gs.18/clones/*/<clone>.fa.gz
#       <contig> is one of the contigs found in:
#               /cluster/store5/gs.18/build35/<chrom>/<contig>/<contig>.fa
#
CHROM=$1
CLONE=$2
CONTIG=$3
TARGET=/iscratch/i/gs.18/build35/maskedContigs/${CONTIG}.fa
FAZ=/iscratch/i/gs.18/build35/clones/${CHROM}/${CLONE}.fa.gz
OOC=/iscratch/i/gs.18/build35/contigOoc10/${CONTIG}.10.ooc
mkdir -p psl/${CONTIG}
if [ ! -s ${FAZ} ]; then
        echo "Can not find: ${FAZ}"
        exit 255
fi
if [ ! -s ${TARGET} ]; then
        echo "Can not find: ${TARGET}"
        exit 255
fi
if [ ! -s ${OOC} ]; then
        echo "Can not find: ${OOC}"
        exit 255
fi
zcat ${FAZ} > /tmp/${CLONE}.fa
$HOME/bin/i386/psLayout ${TARGET} \
        /tmp/${CLONE}.fa genomic ${OOC} psl/${CONTIG}/${CLONE}.psl
RET=$?
rm -f /tmp/${CLONE}.fa
exit ${RET}
'_EOF_'
    # << this line makes emacs coloring happy
    chmod +x runPsLayout.sh

    #	make up a listing of chrom, clone, contig from:
    grep -v "^#" disburseEm.list | sed -e "s/.fa.gz//" > chr.clone.contig.list
    wc -l chr.clone.contig.list
    #	26872 chr.clone.contig.list
    awk '{
printf "./runPsLayout.sh %s %s %s {check out line+ psl/%s/%s.psl}\n",
        $1, $2, $3, $3, $2
}' chr.clone.contig.list > jobList
    # << this line makes emacs coloring happy
    #	To do a quick test, run just chr22:
    grep -v "^22" chr.clone.contig.list | awk '{
printf "./runPsLayout.sh %s %s %s {check out line+ psl/%s/%s.psl}\n",
        $1, $2, $3, $3, $2
}' > jobList
    para create jobList
    para try ... check ... etc ...
    #	One run on chr22 took:
# Completed: 561 of 561 jobs
# CPU time in finished jobs:     927068s   15451.14m   257.52h   10.73d  0.029 y
# IO & Wait Time:                  6295s     104.91m     1.75h    0.07d  0.000 y
# Average job time:                1664s      27.73m     0.46h    0.02d
# Longest job:                    69745s    1162.42m    19.37h    0.81d
# Submission to last job:         69780s    1163.00m    19.38h    0.81d


    #	put the results together, filter, lift and load:
    cd /cluster/data/hg17/bed/contig_overlaps/psl
    pslSort dirs raw.psl tmp N*
    pslReps -singleHit raw.psl repsSingle.psl /dev/null
    liftUp chr22.psl /cluster/data/hg17/jkStuff/liftAll.lft \
	warn repsSingle.psl
    hgLoadPsl -table=cloneTest hg17 chr22.psl

    #	There are a number of clones listed in the sequence.inf file
    #	as status W with names beginning AACC AADB AADC AADD
    #	These are the Whole shotgun assemblies for the Celera genome.
    #	A few of them were used in the assembly of the NCBI genome, namely:
./11/AADB01066164.1.fa.gz
./11/AADC01095577.1.fa.gz
./11/AADD01116830.1.fa.gz
./11/AADD01118406.1.fa.gz
./11/AADD01116787.1.fa.gz
./11/AADD01112371.1.fa.gz
./11/AADD01116788.1.fa.gz
./11/AADD01115518.1.fa.gz
./11/AADD01118410.1.fa.gz
./11/AADD01117999.1.fa.gz
./21/AADD01172789.1.fa.gz
./21/AADD01172788.1.fa.gz
./21/AADD01209098.1.fa.gz
./21/AADD01172902.1.fa.gz
    #	And these have been distributed properly in their corresponding
    #	chromosome.  The rest of them, 26, all with names starting AACC are in
    #	the directory here: celeraOnly

    #	To run the unPlaced alignments.
    #	Prepare scratch and iscratch
    ssh eieio
    mkdir /cluster/bluearc/scratch/hg/gs.18/build35/clones/unPlaced
    rsync -arlv /cluster/data/hg17/bed/contig_overlaps/unPlaced/ \
	/cluster/bluearc/scratch/hg/gs.18/build35/clones/unPlaced
    #	request scratch sync to cluster admins

    ssh kkr1u00
    mkdir /iscratch/i/gs.18/build35/clones/unPlaced
    rsync -arlv /cluster/data/hg17/bed/contig_overlaps/unPlaced/ \
	/iscratch/i/gs.18/build35/clones/unPlaced
    /cluster/bin/iSync

    ssh hgwdev
    cd /cluster/data/hg17/bed/contig_overlaps/unPlacedBatch
    #	There are too many to try them all, obtain guildelines from hg16
    #	of clone to contig mapping:
    hgsql -N -e "select name,chrom from clonePos;" hg16 > hg16.clone.chrom
    hgsql -N -e "select contig,chrom from ctgPos;" hg16 > hg16.contig.chrom

    ssh kk
    mkdir /cluster/data/hg17/bed/contig_overlaps/unPlacedBatch
    cd /cluster/data/hg17/bed/contig_overlaps/unPlacedBatch
    ls ../unPlaced | sed -e "s/.fa.gz//" > unPlaced.clone.list
    wc -l unPlaced.clone.list
    #	19836 unPlaced.clone.list
    ls -1S /scratch/hg/gs.18/build35/maskedContigs > contig.list
    wc -l contig.list
    #	380 contig.list

    cat << '_EOF_' > runPsLayout.sh
#!/bin/sh
#       kkiPsLayout.sh <clone> <contig>
#       <clone> is one of the .fa.gz files in
#               /scratch/hg/gs.18/build35/clones/unPlaced
#               without the .fa.gz extension
#       <contig> is one of the contigs found in:
#               /iscratch/i/gs.18/build35/maskedContigs
#
CLONE=$1
CONTIG=$2
TARGET=/iscratch/i/gs.18/build35/maskedContigs/${CONTIG}.fa
FAZ=/scratch/hg/gs.18/build35/clones/unPlaced/${CLONE}.fa.gz
OOC=/iscratch/i/gs.18/build35/contigOoc10/${CONTIG}.10.ooc
mkdir -p psl/${CONTIG}
if [ ! -s ${FAZ} ]; then
        echo "Can not find: ${FAZ}"
        exit 255
fi
if [ ! -s ${TARGET} ]; then
        echo "Can not find: ${TARGET}"
        exit 255
fi
if [ ! -s ${OOC} ]; then
        echo "Can not find: ${OOC}"
        exit 255
fi
zcat ${FAZ} > /tmp/${CLONE}.fa
$HOME/bin/i386/psLayout ${TARGET} \
        /tmp/${CLONE}.fa genomic ${OOC} psl/${CONTIG}/${CLONE}.psl
RET=$?
rm -f /tmp/${CLONE}.fa
exit ${RET}
'_EOF_'
    # << this line makes emacs coloring happy
    chmod +x runPsLayout.sh

    cat << '_EOF_' > gsub
#LOOP
./runPsLayout.sh $(path1) $(path2) {check out line+ psl/$(path2)/$(path1).psl}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    gensub2 unPlaced.clone.list contig.list gsub jobList

   # XXXX - some time later ... 2004-07-12
    # Bringing this sequence to a close.  Difficulties encountered:
    #	Placed clones that did not survive the psLayout filter:
    #	AC006040.3 AC006328.5 AC007039.6 AC007241.3 AC007965.3
    #	AC009947.2 AC010682.2 AC012005.4 AC016707.2 AC016728.4
    #	AC016752.2 AC017005.7 AC025226.4 AC025246.6 AC055713.29
    #	AC068541.7 AC068601.8 AC068704.4 AC073649.3 AC073962.5
    #	AC091175.11 AC095381.1 AC104597.3 AC130223.2 AC130814.3
    #	AC133883.6 AC139103.3 AF003627.3 AF135405.3 AL021878.2
    #	AL137064.6 AL356803.2 AL390801.4 AL591480.8 AL901608.1
    #	AP005814.2 BX322790.2 Z84489.1 Z84814.1
    #	And placed clones that broken into two pieces during their
    #	psLayout alignment:
    #	AC006982.3 AC007742.4 AC023342.3 AC024183.4 AC025735.4
    #	AC095380.1 AL646104.4 BX293536.4
    #	For the above clones, their assignments in ref_placed.agp were
    #	used instead of trying to adjust the psLayout process.

    #	The PAR clones are a problem.  They were placed properly, but
    #	during their load with hgClonePos there was a warning issued
    #	about their dual existance.  hgClonePos said they were only
    #	going to be placed on chrX and not on chrY.  However in the
    #	browser when chrY is viewed it issues errors about these not
    #	having proper coordinates in the clonePos table.  These were
    #	removed from the coverage track to eliminate that error.
    #	AL954722.18 BX537334.4 BX000483.7 BX908402.3 BX649635.3 BX119919.5
    #	AC079176.15 AC097314.27 AC006209.25 AJ271735.1 AJ271736.1
    #
    #	And finally, after many different types of alignment attempts,
    #	there remain 1489 un-placed clones that could not be located.

    #	While trying to figure out which contigs many clones belonged
    #	to, the following cluster run script was used to take a survey
    #	using blat:
#!/bin/sh
#       runBlat.sh <clone> <contig>
#       <clone> is one of the .fa.gz files in
#               /scratch/hg/gs.18/build35/clones/
#               without the .fa.gz extension
#       <contig> is one of the contigs found in:
#               /iscratch/i/gs.18/build35/maskedContigs
#   
# ./runBlat.sh unPlaced/AB000876.1.fa.gz NT_005612.fa {check out line+
# psl/NT_005612.fa/unPlaced/AB000876.1.fa.gz.psl}
#
HERE=`pwd`
CLONE=$1 
CLONEDIR=`dirname ${CLONE}`
CLONENAME=`basename ${CLONE}`
CLONESRC=/iscratch/i/gs.18/build35/clones/${CLONE}.fa.gz
CONTIG=$2
CONTIGBASE=${CONTIG/.fa/}
TARGET=/iscratch/i/gs.18/build35/maskedContigs/${CONTIG}
if [ ! -s ${CLONESRC} ]; then
        echo "Can not find: ${CLONESRC}" 1>/dev/stderr
        exit 255
fi
if [ ! -s ${TARGET} ]; then
        echo "Can not find: ${TARGET}" 1>/dev/stderr
        exit 255
fi
mkdir -p /tmp/${CLONEDIR}/${CLONENAME}
zcat ${CLONESRC} > /tmp/${CLONEDIR}/${CLONENAME}/${CLONENAME}.fa
cd /tmp/${CLONEDIR}
/cluster/data/hg17/bed/contig_overlaps/FfaSplit/faToFfa ${CLONENAME}
ECOUNT=`cat error.convert | wc -l`
if [ "${ECOUNT}" -ne 0 ]; then
        echo "Error during faToFfa, error.convert not empty" 1>/dev/stderr
        exit 255
fi
rm -f error.convert
B=${CLONENAME/\.*/}
cd /tmp/${CLONEDIR}/${CLONENAME}
faSplit byname ${CLONENAME}.fa .
RET=0
export RET
for F in ${CLONENAME}_*.fa
do
    FA=${F/_*.fa/}
    A=${FA/.[0-9]*/}
    P=${F/.fa/}
    N=${P##*_}
    rm -f t.fa
    mv ${F} t.fa
    cat t.fa | faSplit -oneFile size stdin 3000 ${A}_${N}
    rm -f t.fa
    blat ${TARGET} ${A}_${N}.fa -ooc=/scratch/hg/h/11.ooc ${A}_${N}.psl \
        -t=dna -q=dna -fastMap -noHead
    RET=$?
    if [ "$RET" -ne 0 ]; then
        echo "Error during blat ${TARGET} ${A}_${N}.fa" 1>/dev/stderr
        break
    fi
done
rm -f ${CLONENAME}.fa
rm -f ${B}_*.fa
cd ${HERE}
mkdir -p psl/${CONTIGBASE}
sed -e "s/${A}/${CLONENAME}/" /tmp/${CLONEDIR}/${CLONENAME}/*.psl > \
        psl/${CONTIGBASE}/${CLONENAME}.psl
rm -f /tmp/${CLONEDIR}/${CLONENAME}/*.psl
rmdir --ignore-fail-on-non-empty /tmp/${CLONEDIR}/${CLONENAME}
rmdir --ignore-fail-on-non-empty /tmp/${CLONEDIR}
exit ${RET}

    #	The alignment with psLayout were done with the following cluster
    #	run script:

#!/bin/sh
#       kkiPsLayout.sh <clone> <contig>
#       <clone> is one of the .fa.gz files in
#               /scratch/hg/gs.18/build35/clones/unPlaced
#               without the .fa.gz extension
#       <contig> is one of the contigs found in:
#               /iscratch/i/gs.18/build35/maskedContigs
#
# ./runPsLayout.sh unPlaced/AP001966.2 NT_016354 {check out exists
# psl/NT_016354/AP001966.2.psl}
#
HERE=`pwd`
CLONE=$1
CONTIG=$2
CLONEDIR=`dirname ${CLONE}`
CLONENAME=`basename ${CLONE}`
RESULT=psl/${CONTIG}/${CLONENAME}.psl
CLONESRC=/iscratch/i/gs.18/build35/clones/${CLONE}.fa.gz
TARGET=/iscratch/i/gs.18/build35/maskedContigs/${CONTIG}.fa
OOC=/iscratch/i/gs.18/build35/contigOoc10/${CONTIG}.10.ooc
if [ ! -s ${CLONESRC} ]; then
        echo "Can not find: ${CLONESRC}" 1>/dev/stderr
        exit 255
fi
if [ ! -s ${TARGET} ]; then
        echo "Can not find: ${TARGET}" 1>/dev/stderr
        exit 255
fi
if [ ! -s ${OOC} ]; then
        echo "Can not find: ${OOC}" 1>/dev/stderr
        exit 255
fi
mkdir -p /tmp/${CONTIG}/${CLONEDIR}/${CLONENAME}
zcat ${CLONESRC} > /tmp/${CONTIG}/${CLONEDIR}/${CLONENAME}.fa
cd /tmp/${CONTIG}
/cluster/data/hg17/bed/contig_overlaps/FfaSplit/faToFfa ${CLONEDIR}
cd ${HERE}
mkdir -p psl/${CONTIG}
$HOME/bin/i386/psLayout ${TARGET} /tmp/${CONTIG}/${CLONEDIR}/${CLONENAME}.fa genomic ${OOC} ${RESULT}
RET=$?
rm -f /tmp/${CONTIG}/${CLONEDIR}/${CLONENAME}.fa /tmp/${CONTIG}/error.convert
rmdir --ignore-fail-on-non-empty /tmp/${CONTIG}/${CLONEDIR}/${CLONENAME}
rmdir --ignore-fail-on-non-empty /tmp/${CONTIG}/${CLONEDIR}/
rmdir --ignore-fail-on-non-empty /tmp/${CONTIG}
exit ${RET}

# BUILD KNOWN GENES TABLES (DONE 6/8/04 Fan)

  Build sp040515 and proteins040515 DBs first.
  
  hgsql hg17 -e "create database kgHg17"
  
  cd /cluster/store6/kgDB/bed
  mkdir kgHg17
  cd /cluster/store6/kgDB/bed/kgHg17

  ~/src/hg/protein/KGprocess.sh kgHg17 hg17 040515
  
  The script was run successfully with the last message:

  	Tue Jun  8 15:36:52 PDT 2004 DONE 

  After initial inspection of tables in kgHg17, do the following
  from mySql prompt:

  alter table kgHg17.cgapAlias rename as hg17.cgapAlias;
  alter table kgHg17.cgapBiocDesc rename as hg17.cgapBiocDesc;
  alter table kgHg17.cgapBiocPathway rename as hg17.cgapBiocPathway;
  alter table kgHg17.dupSpMrna rename as hg17.dupSpMrna;
  alter table kgHg17.keggMapDesc rename as hg17.keggMapDesc;
  alter table kgHg17.keggPathway rename as hg17.keggPathway;
  alter table kgHg17.kgAlias rename as hg17.kgAlias;
  alter table kgHg17.kgProtAlias rename as hg17.kgProtAlias;
  alter table kgHg17.kgXref rename as hg17.kgXref;
  alter table kgHg17.knownGene rename as hg17.knownGene;
  alter table kgHg17.knownGeneLink rename as hg17.knownGeneLink;
  alter table kgHg17.knownGeneMrna rename as hg17.knownGeneMrna;
  alter table kgHg17.knownGenePep rename as hg17.knownGenePep;
  alter table kgHg17.mrnaRefseq rename as hg17.mrnaRefseq;
  alter table kgHg17.spMrna rename as hg17.spMrna;

  hg17.knownGene has 43,401 entries and hg16.knownGene has 43,232 entries.
  and running featireBits shows:
  
   	featureBits hg17 knownGene
   	63983072 bases of 2866216770 (2.232%) in intersection
   
   	featureBits hg16 knownGene
   	63781799 bases of 2865248791 (2.226%) in intersection
  
  Connect to genome-testdb and use hgcentraltest DB.
  Add a new entry in gdbPdb table:
 
        insert into gdbPdb values('hg17', 'proteins040515');


# CREATE LINEAGE-SPECIFIC REPEATS FOR BLASTZ WITH ZEBRAFISH
# (DONE, 2004-06-08, hartera)
    # Treat all repeats as lineage-specific
    mkdir /iscratch/i/gs.18/build35/linSpecRep.notInZebrafish
    foreach f (/iscratch/i/gs.18/build35/rmsk/chr*.fa.out)
 cp -p $f /iscratch/i/gs.18/build35/linSpecRep.notInZebrafish/$f:t:r:r.out.spec
    end
    iSync
  

# PREP FOR LIFTOVER CHAINS TO THIS ASSEMBLY (2004-06-10 kate)

    # split into 3K chunks
    ssh eieio
    set liftDir = /cluster/data/hg17/bed/liftOver/liftSplit
    mkdir -p $liftDir
    cd $liftDir
cat > split.csh << 'EOF'
    set splitDir = /iscratch/i/hg17/liftOver/split
    mkdir -p $splitDir
    set liftDir = /cluster/data/hg17/bed/liftOver/liftSplit
    foreach n (`ls /cluster/data/hg17/nib`)
        set c = $n:r
        echo $c
        faSplit -lift=$liftDir/$c.lft size \
            /cluster/data/hg17/$d/$c.fa -oneFile 3000 $splitDir/$c
    end
'EOF'
# << for emacs
    csh split.csh >&! split.log &
    tail -100f split.log

    ssh kkr1u00
    iSync


# STS MARKERS (DONE 2004-07-21 kate)
# MANUAL UPDATE OF D21S168 and D21S167 (DONE, 2005-02-11, hartera)
# FILTERED OUT noOoc ALIGNMENTS WITH tBaseInsert >=1000 
# (DONE, 2005-02-17, hartera) AND RELOADED stsMap, stsInfo2 and all_sts_seq
# DATABASE TABLES AFTER ADDING FILTERED ALIGNMENTS TO all_sts_seq AND 
# REMOVING DATA FROM stsMap and stsInfo2 FOR THE MARKERS REMOVED FROM THE 
# FILTERED SET (DONE, 2005-02-18, hartera)
# UPDATE PSL ALIGNMENTS FOR D21S167 and D21S168 AND RELOAD INTO all_sts_seq
# (DONE, 2005-02-23, hartera)
# UPDATED stsAlias TABLE REMOVING OF FILTERED ALIGNMENTS (2005-02-24, hartera)

    # Terry's sts.9 dir is in /cluster/store5/sts.2004-07.old
    # remove this after verifying the newer version

   # update from NCBI (booch)
    ssh eieio
    # use store5 for space
    mkdir -p /cluster/store5/sts.2004-07
    ln -s /cluster/store5/sts.2004-07 /cluster/data/ncbi
    ln -s /cluster/data/ncbi/sts.2004-07 sts.9
    cd /cluster/data/ncbi/sts.2004-07
    wget ftp://ftp.ncbi.nih.gov/repository/dbSTS/dbSTS.sts
    wget ftp://ftp.ncbi.nih.gov/repository/dbSTS/dbSTS.aliases
    wget ftp://ftp.ncbi.nih.gov/blast/db/FASTA/sts.gz
    gunzip sts.gz
    mv sts dbSTS.fa

    # incremental update from previous build
    # NOTE: could mysql dump this, unless hand-updated (like hg16)
    # First - copy from Terry's dir
    ssh eieio
    ln -s /cluster/store1/sts.8 /cluster/data/ncbi
    cd /cluster/data/ncbi/sts.9

    # this time, snag from Terry's dir
    cd /cluster/data/ncbi/sts.9
    cp -p ~booch/tracks/update/all.STS.fa.prev .
    cp -p ~booch/tracks/update/stsInfo2.bed stsInfo2.bed.prev

    # Convert dbSTS.fa file to easier reading format, and get accessions
    /cluster/bin/scripts/convertGbFaFile dbSTS.fa > dbSTS.convert.fa
    grep ">" dbSTS.convert.fa | cut -f 2 -d ">" > dbSTS.acc

    # NOTE: updateStsInfo creates new stsInfo2.bed, all.primers, 
    #   all.STS.fa, stsAlias.bed files 
    updateStsInfo -verbose=1 -gb=dbSTS.acc stsInfo2.bed.prev all.STS.fa.prev \
                    dbSTS.sts dbSTS.aliases dbSTS.convert.fa new
    # 129991  SWXD2599        99622   (0) not in dbSTS anymore
    # 166473  D3S3812 154523  (0) not in dbSTS anymore
    # 185776  RH83562 209614  (0) not in dbSTS anymore

    mv new.info stsInfo2.bed
    mv new.primers all.primers
    mv new.alias stsAlias.bed
    mv new.fa all.STS.fa

    # get list of all STS id's in the fasta file
    sed -n 's/^>\([0-9][0-9]*\) .*/\1/p' all.STS.fa | sort -n >  all.STS.id
    wc -l all.STS.id
        # 92674 total sequences
    /cluster/bin/scripts/convertPrimerToFA all.primers > all.primers.fa

    # Copy stsInfo2.bed and stsAlias.bed to data directory becuase
    # these will be loaded into the database later
    mkdir -p /cluster/data/hg17/bed/sts
    cp stsInfo2.bed /cluster/data/hg17/bed/sts/
    cp stsAlias.bed /cluster/data/hg17/bed/sts/

    # Create sts sequence alignments
    mkdir -p /cluster/bluearc/sts.9/sts.split
    faSplit sequence all.STS.fa 50 /cluster/bluearc/sts.9/sts.split/sts
    cp /cluster/data/ncbi/sts.9/all.STS.fa /cluster/bluearc/sts.9

    # create small ooc file to use with alignments (if not existing)
    # NOTE: these were just used for experimenting; weren't used in
    # final runs
    ssh kolossus
    cd /cluster/data/hg17/bed/sts
    ls /cluster/bluearc/hg17/bothMaskedNibs/chr*.nib > nib.lst
    blat nib.lst /dev/null /dev/null \
        -tileSize=11 -makeOoc=/cluster/bluearc/hg/h/11.4096.ooc -repMatch=4096
    blat nib.lst /dev/null /dev/null \
        -tileSize=11 -makeOoc=/cluster/bluearc/hg/h/11.16384.ooc -repMatch=16384

    ssh kk
    cd /cluster/data/hg17/bed/sts
    mkdir run
    cd run
    ls -1S /scratch/hg/hg17/maskedContigs/*.fa > contigs.lst
    ls -1S /cluster/bluearc/sts.9/sts.split/sts*.fa > sts.lst
    mkdir -p /cluster/bluearc/hg17/sts/sts/out
    foreach f (`cat sts.lst`)
        set d = $f:t:r
        mkdir /cluster/bluearc/hg17/sts/sts/out/$d
    end

    # create alignments
cat > template << 'EOF'
#LOOP
/cluster/bin/i386/blat $(path1) $(path2) -ooc=/cluster/bluearc/hg/h/11.ooc -stepSize=5 {check out line+ /cluster/bluearc/hg17/sts/sts/out/$(root2)/$(root1).$(root2).psl}
#ENDLOOP
'EOF'
# << for emacs

    gensub2 contigs.lst sts.lst template jobList
    para create jobList
        # 17860 jobs
    para try
    para check
    para push
# CPU time in finished jobs:     216985s    3616.41m    60.27h    2.51d  0.007 y
# IO & Wait Time:                 48790s     813.17m    13.55h    0.56d  0.002 y
# Average job time:                  15s       0.25m     0.00h    0.00d
# Longest job:                      267s       4.45m     0.07h    0.00d
# Submission to last job:          2228s      37.13m     0.62h    0.03d

    # Compile sts sequence results
    ssh kolossus
    cd /cluster/bluearc/hg17/sts/sts
    pslSort dirs raw.psl temp out/*
    rm -rf temp
    pslReps -nearTop=0.0001 -minCover=0.6 -minAli=0.8 -noIntrons raw.psl \
	stsMarkers.psl /dev/null
            # Processed 7121016 alignments
    #cp stsMarkers.psl /cluster/data/hg17/bed/sts/run

    # Lift them and get them ready to combine with primer alignments
    #cd  /cluster/data/hg17/bed/sts/run
    #liftUp -nohead /cluster/data/hg17/bed/sts/run/stsMarkers.lifted.psl \
    liftUp -nohead stsMarkers.lifted.psl \
        /cluster/data/hg17/jkStuff/liftAll.lft warn stsMarkers.psl

    # missing some utilities for kolossus, so switch to fileserver
    # NOTE: probably no longer true -- try on kolossus next time
    ssh kksilo
    cd /cluster/bluearc/hg17/sts/sts
    /cluster/bin/scripts/extractPslInfo stsMarkers.lifted.psl
        # creates <file>.initial
    /cluster/bin/scripts/findAccession -agp stsMarkers.lifted.psl.initial \
	/cluster/data/hg17
            # "Could not open /cluster/data/hg17/Y/chrY_random.agp" etc.
            # Looks like it trys all _randoms (even one's that don't
            # exist/aren't needed
            # creates <file>.acc
    #rm stsMarkers.lifted.psl.initial
    sort -k 4n stsMarkers.lifted.psl.initial.acc > stsMarkers.final
    #rm stsMarkers.lifted.psl.initial.acc
    #cp stsMarkers.final stsMarkers.lifted.psl.initial /cluster/data/hg17/bed/sts
    # determine found markers (4th field in file)
    cut -f 4 stsMarkers.final | sort -n -u > stsMarkers.found
    wc -l stsMarkers.found
        #   89532 stsMarkers.found
        # out of 92674 total sequences

    # extract sequences for markers not yet found, and
    # blat w/o ooc to try to place more
    comm -1 -3  stsMarkers.found /cluster/data/ncbi/sts.9/all.STS.id \
                > stsMarkers.notFound
    wc -l stsMarkers.notFound
        # 3142 stsMarkers.notFound
    faSomeRecords /cluster/data/ncbi/sts.9/all.STS.fa stsMarkers.notFound \
                notFound.STS.fa
    mkdir /cluster/bluearc/sts.9/sts.splitNotFound
    faSplit sequence notFound.STS.fa 20 \
                /cluster/bluearc/sts.9/sts.splitNotFound/sts

    # blat with 11.ooc misses alignments, so reblat w/o the
    # sequences that aren't found
    # NOTE: filtering produces yield of only 149 markers placed (out of 3142).
    # not enough to justify this step next time
    ssh kk
    cd /cluster/data/hg17/bed/sts
    mkdir run.noOoc
    cd run.noOoc
    ls -1S /scratch/hg/hg17/maskedContigs/*.fa > contigs.lst
    ls -1S /cluster/bluearc/sts.9/sts.splitNotFound/sts*.fa > sts.lst
    mkdir -p /cluster/bluearc/hg17/sts/sts/out.noOoc

    foreach f (`cat sts.lst`)
        set d = $f:t:r
        mkdir /cluster/bluearc/hg17/sts/sts/out.noOoc/$d
    end

cat > template << 'EOF'
#LOOP
/cluster/bin/i386/blat $(path1) $(path2) -stepSize=5 {check out line+ /cluster/bluearc/hg17/sts/sts/out.noOoc/$(root2)/$(root1).$(root2).psl}
#ENDLOOP
'EOF'
# << for emacs

    gensub2 contigs.lst sts.lst template jobList
    para create jobList
        # 7220 jobs written to batch
    para try
    para check

    # process this set of alignments
    ssh kolossus
    cd /cluster/bluearc/hg17/sts/sts
    pslSort dirs raw.noOoc.psl temp out.noOoc/*

    rm -rf temp
    pslReps -nearTop=0.0001 -minCover=0.6 -minAli=0.8 -noIntrons \
        raw.noOoc.psl stsMarkers.noOoc.psl /dev/null
            # Processed 4254094 alignments
    #cp stsMarkers.psl /cluster/data/hg17/bed/sts/run

    # Lift them and get them ready to combine with primer alignments
    liftUp -nohead stsMarkers.noOoc.lifted.psl \
        /cluster/data/hg17/jkStuff/liftAll.lft warn stsMarkers.noOoc.psl

    /cluster/bin/scripts/extractPslInfo stsMarkers.noOoc.lifted.psl
        # creates <file>.initial
    /cluster/bin/scripts/findAccession -agp \
        stsMarkers.noOoc.lifted.psl.initial /cluster/data/hg17
            # "Could not open /cluster/data/hg17/Y/chrY_random.agp" etc.
            # Looks like it trys all _randoms (even one's that don't
            # exist/aren't needed
            # creates <file>.acc
    #rm stsMarkers.lifted.psl.initial
    mv stsMarkers.final stsMarkers.ooc.final
    sort -k 4n stsMarkers.noOoc.lifted.psl.initial.acc > stsMarkers.extra
    sort -k 4n stsMarkers.lifted.psl.initial.acc \
                stsMarkers.noOoc.lifted.psl.initial.acc > stsMarkers.final

    # determine found markers (4th field in file)
    cut -f 4 stsMarkers.final | sort -n -u > stsMarkers.found
    wc -l stsMarkers.found
        #  89681 stsMarkers.found
    cut -f 4 stsMarkers.extra | sort -n -u > stsMarkers.extra.found
    wc -l stsMarkers.extra.found
        #   149 out of 3142 attempted
        # out of 92674 total sequences
    cp stsMarkers.final stsMarkers.lifted.psl stsMarkers.*lifted.psl.initial* stsMarkers.found \
                /cluster/data/hg17/bed/sts
    # Alignments from noOoc set were not added to all_sts_seq but info for the markers
    # is in stsMap and stsInfo2. Some of the alignments are bad so filter by removing
    # all alignments from noOoc psl file where tBaseInsert >=1000. Add the remaining 
    # alignments to the set of final alignments for stsMarkers. The information for the  
    # removed markers from the filtered set was also removed from stsMap and stsInfo2.
    # (DONE, 2005-02-17, hartera)
    ssh eieio
    cd /cluster/data/hg17/bed/sts/fix
    cp /cluster/bluearc/hg17/sts/sts/stsMarkers.noOoc.lifted.psl .
    awk '{if ($8 < 1000) print;}' stsMarkers.noOoc.lifted.psl > stsMarkers.noOoc.lifted.filt1000.psl
    wc -l *.filt*.psl
    # 254    5334   26384 stsMarkers.noOoc.lifted.filt1000.psl
    sort -k 4n /cluster/bluearc/hg17/sts/sts/stsMarkers.noOoc.lifted.psl.initial.acc > stsMarkers.extra
    awk '{print $4;}' stsMarkers.extra | sort -n | uniq >  extra.ids
    # in psl file, the ids are the 10th field
    awk '{print $10;}' stsMarkers.noOoc.lifted.psl | sort -n | uniq \
        > noOoc.ids
    diff extra.ids noOoc.ids
    # there is no difference as expected
    # get list of IDs from filtered file, filter < 1000
    awk '{print $10;}' stsMarkers.noOoc.lifted.filt1000.psl \
        | sort -n | uniq > filt1000.ids
    foreach i (`cat filt1000.ids`)
     awk 'BEGIN {OFS="\t"} \
         {if ($4 == "'$i'") print $1, $2, $3, $4, $5, $6, $7}' \
         stsMarkers.extra >> stsMarkers.extra.filt1000
    end
    cp ../stsMarkers.final stsMarkers.final
  #  cat stsMarkers.extra.filt1000 >> stsMarkers.final2 
     # need to filter stsMarkers.final not just cat this on the end
    # get list of alignments with tBaseInsert >= 1000 and remove these
    cd /cluster/data/hg17/bed/sts/fix
    awk '{if ($8 >= 1000) print;}' stsMarkers.noOoc.lifted.psl > stsMarkers.noOoc.lifted.filtToRemove.psl
    wc -l *.filt*.psl
    # 254 stsMarkers.noOoc.lifted.filt1000.psl
    # 249 stsMarkers.noOoc.lifted.filt500.psl
    # 448 stsMarkers.noOoc.lifted.filtToRemove.psl
    # get list of IDs that need to be removed
    awk '{print $10;}' stsMarkers.noOoc.lifted.filtToRemove.psl | sort -n \
        | uniq  > noOoc.IdsToRemove.txt
    # get chrom and co-ordinates for IDs to be removed 
    awk 'BEGIN {OFS = "\t"} {print $14,$16,$17,$10}' \
        stsMarkers.noOoc.lifted.filtToRemove.psl | sort | uniq \
        > sts.noOoc.filtToRemove.coords
    # checked that the stsMarkers.final contain the noOoc alignments
    # wrote perl script to remove lines with these IDs from stsMarkers.final
cat << '_EOF_' > removeIds.pl
#!/usr/bin/perl -w
use strict;

my $ids = $ARGV[0];
my $file = $ARGV[1];
# list of IDs with chrom and coords to remove
open(IDS, $ids) || die "Can not open $ids: $!\n";
# file for removal of IDs
open(FILE, $file) || die "Can not open $file: $!\n";
open(OUT, ">removed.txt") || die "Can not create removed.txt: $!\n";

my %idsHash;

while (<IDS>) {
   chomp;
   my @a = split(/\t/);

   my $chr = $a[0];
   my $st = $a[1];
   my $end = $a[2];
   my $id = $a[3];
   my $key = $id."_".$chr . "_" . $st . "_" . $end;
   $idsHash{$key}->{chrom} = $chr;
   $idsHash{$key}->{start} = $st;
   $idsHash{$key}->{end} = $end;
}
close IDS;

while (<FILE>) {
   chomp;
   my $l = $_;
   my $found = "FALSE";
   my @f = split(/\t/, $l);
   foreach my $k (keys(%idsHash)) {
      # if the id is contained in the key
      if ($k =~ /^$f[3]/) {
         my $c = $idsHash{$k}->{chrom};
         my $s = $idsHash{$k}->{start};
         my $e = $idsHash{$k}->{end};
         if ($f[0] eq $c && $f[1] == $s && $f[2] == $e) {
             print OUT "$c\t$s\t$e\t$f[3]\n";
             $found = "TRUE";
         }
      }
   }
   if ($found eq "FALSE") {
      print "$l\n";
   }
}
'_EOF_'
    chmod +x removeIds.pl
    perl removeIds.pl sts.noOoc.filtToRemove.coords stsMarkers.final \
         > stsMarkers.final.new
    wc -l stsMarkers.final*
    # 92338 stsMarkers.final
    # 91890 stsMarkers.final.new
    # There are 448 ids and sets of co-ordinates in list of Ids to remove
    # check that stsMarkers.final.new contains all the alignments that
    # are in filtered set: stsMarkers.noOoc.lifted.filt1000.psl
    awk 'BEGIN {OFS = "\t"} {print $14,$16,$17,$10}' \
        stsMarkers.noOoc.lifted.filt1000.psl | sort | uniq \
        > sts.noOoc.filt1000.coords
    awk 'BEGIN {OFS = "\t"} {print $1,$2,$3,$4}' \
        stsMarkers.final.new | sort | uniq \
        > sts.finalnew.coords
    diff sts.finalnew.coords sts.noOoc.filt1000.coords > finalnewvsfilt1000
    grep '>' finalnewvsfilt1000
    # there is nothing in sts.noOoc.filt1000.coords not found in the 
    # sts.finalnew.coords file therefore this contains all the alignments
    # from the filtered noOoc file.
    cp ../primers/primers.final .
    awk '{print $4}' primers.final | sort | uniq > primers.ids
    awk '{print $4}' stsMarkers.final.new | sort | uniq > stsfinal.new.ids


    # primers
    ssh eieio
    cd /cluster/data/ncbi/sts.9
    # strip out N's and wobbles (KS) from primers, as isPcr
    # can't currently handle them
    # strip out primers < 10 as isPcr can't handle them
    awk '$0 !~ /[^ACGT0-9\-\t]/ && (length($2) > 10) && (length($3) > 10) {printf "dbSTS_%s\t%s\t%s\n", $1,$2,$3}' \
                all.primers > all.primers.ispcr
    mkdir -p /cluster/bluearc/sts.9/primers
    cd /cluster/bluearc/sts.9/primers
    split -l 2000 /cluster/data/ncbi/sts.9/all.primers.ispcr primers_

    ssh kk
    cd /cluster/data/hg17/bed/sts
    mkdir primers
    cd primers
    mkdir run
    cd run
    ls -1S /scratch/hg/gs.18/build35/maskedContigs/*.fa > contigs.lst
    ls -1S /cluster/bluearc/sts.9/primers/primers_* > primers.lst
    mkdir -p /cluster/bluearc/hg17/sts/primers/out

cat > template << 'EOF'
#LOOP
/cluster/home/kate/bin/i386/isPcr -out=psl -minPerfect=2 -maxSize=5000 -tileSize=10 -ooc=/scratch/hg/h/10.ooc  -stepSize=5 $(path1) $(path2) {check out line /cluster/bluearc/hg17/sts/primers/out/$(root1)_$(root2).psl}
#ENDLOOP
'EOF'
# << for emacs

    gensub2 contigs.lst primers.lst template jobList
    para create jobList
	# 26980 jobs
    para try
    para check
    para push

    #Completed: 26953 of 26980 jobs
    #Crashed: 27 jobs
    #CPU time in finished jobs:    1130353s   18839.22m   313.99h   13.08d  0.036 y
    #IO & Wait Time:                 86067s    1434.44m    23.91h    1.00d  0.003 y
    #Average job time:                  45s       0.75m     0.01h    0.00d
    #Longest job:                     1255s      20.92m     0.35h    0.01d
    #Submission to last job:          2762s      46.03m     0.77h    0.03d

    # 27 jobs seg faulted due to -minPerfect=2.
    # Looks like a bug in isPcr -- till it's fixed,
    # we'll rerun with -minPerfect=5 (Terry determined they
    # all complete with this (he used 3, 4, or 5, tuned individually
    # for each job, but just using 5 should be adequate and 
    # less labor-intensive).
    # NOTE: isPcr bug is fixed -- this shouldn't be necessary for
    # next run

    para crashed | grep isPcr | sed 's/minPerfect=2/minPerfect=5/' \
        > jobList.minPerfect5
    para create jobList.minPerfect5
        # 28 jobs
    # repeat with increasing minPerfect, till all complete succesfully

    # Filter output file quickly based on simple parameters
    ssh kolossus
    cd /cluster/bluearc/hg17/sts/primers/
    mkdir -p filter
    pslQuickFilter -minMatch=26 -maxMismatch=5 -maxTinsert=5000 -verbose out/ filter/
	# Note: there will be many messages saying files are empty - this is OK
    pslSort dirs primers.psl.unlifted temp filter

    # filter primer alignments and create not found primer file for ePCR run (booch)
    pslFilterPrimers /cluster/bluearc/hg17/sts/primers/primers.psl.unlifted  \
	/cluster/data/ncbi/sts.9/all.primers primers.filter.unlifted.psl
        # creates $3.notfound.primers
    wc -l primers.filter.unlifted.psl.notfound.primers                   
    # 21919  primers.filter.unlifted.psl.notfound.primers

    # use Greg Schuler's ePCR to attempt alignment of primers missed
    # by isPcr
    mkdir -p /cluster/data/hg17/bed/sts/primers/run.epcr
    mkdir -p /cluster/bluearc/hg17/sts/primers/epcr
    cd /cluster/bluearc/hg17/sts/primers/epcr
    split -l 2500 /cluster/data/hg17/bed/sts/primers/primers.filter.unlifted.psl.notfound.primers  primers_
    cd /cluster/data/hg17/bed/sts/primers/run.epcr
    ls -1S /cluster/bluearc/hg17/sts/primers/epcr/primers_* > primers.lst
    # create contig.lst based on split in build dir
    # NOTE: should probably replace this with something more standard
    # and faster.  Also, this appears to cause load spikes on fileservers.
    # Should get contigs from bluearc, iservers, or cluster local disk
    # At least it's over pretty quick!
    ssh eieio
    cd /cluster/data/hg17/bed/sts/primers/run.epcr
    /cluster/bin/scripts/splitContigList -ncbi /cluster/data/hg17 1
    # next time... ls -1S /cluster/bluearc/hg17/contigs/* > contig.lst (?)
    mkdir -p /cluster/bluearc/hg17/sts/primers/epcr/out

    ssh kk
    cd /cluster/data/hg17/bed/sts/primers/run.epcr

cat > template << 'EOF'
#LOOP
/cluster/bin/scripts/runEpcr $(path1) $(path2) {check out line /cluster/bluearc/hg17/sts/primers/epcr/out/$(root1).$(root2).epcr}
#ENDLOOP
'EOF'
# << for emacs
    gensub2 primers.lst contig.lst template jobList
    para create jobList
	# 3420 jobs
    para try
    para check
    para push

# CPU time in finished jobs:      78897s    1314.95m    21.92h    0.91d  0.003 y
# IO & Wait Time:                254582s    4243.03m    70.72h    2.95d  0.008 y
# Average job time:                  98s       1.63m     0.03h    0.00d
# Longest job:                      647s      10.78m     0.18h    0.01d
# Submission to last job:          1112s      18.53m     0.31h    0.01d

    # merge output
    ssh eieio
    cd /cluster/bluearc/hg17/sts/primers/epcr
    cat out/*.epcr > all.epcr
    wc -l all.epcr
    # 3573 

    # use all.epcr file to re-filter alignemnts and determine which
    # ePCR records to keep
    cp all.epcr /cluster/data/hg17/bed/sts/primers
    cd /cluster/data/hg17/bed/sts/primers
    pslFilterPrimers -epcr=all.epcr -verbose=1 \
        /cluster/bluearc/hg17/sts/primers/primers.psl.unlifted \
	/cluster/data/ncbi/sts.9/all.primers primers.unlifted.epcr.psl

    # convert to PSL and combine with other psl file (this takes a couple hours)
    /cluster/bin/scripts/epcrToHgPsl epcr.not.found \
        /cluster/data/ncbi/sts.9/all.primers /cluster/data/hg17
    cat primers.unlifted.epcr.psl epcr.not.found.psl \
                | sort -k 10n > primers.final.unlifted.psl

    # Fix the query gap lengths so that they match the all.primers.fa 
    #   file lengths
    /cluster/bin/scripts/fixPrimersQueryGaps \
        /cluster/data/ncbi/sts.9/all.primers primers.final.unlifted.psl \
                > primers.final.unlifted.fix.psl

    # lift results from contigs to chrom coordinates, and create final file
    liftUp -nohead /cluster/data/hg17/bed/sts/primers/primers.psl \
            /cluster/data/hg17/jkStuff/liftAll.lft warn \
            primers.final.unlifted.fix.psl
    # Extract relevant info, make alignments unique, and create final file to be merged
    # with full sequence alignments
    /cluster/bin/scripts/extractPslInfo primers.psl
    /cluster/bin/scripts/findAccession -agp primers.psl.initial \
                /cluster/data/hg17
    #rm primers.psl.initial

    /cluster/bin/scripts/getStsId /cluster/data/ncbi/sts.9/stsInfo2.bed \
	primers.psl.initial.acc \
        | sort -k 4n > primers.final
    #rm primers.psl.initial.acc
    wc -l primers.final
    # 314713 primers.final

    # Merge primer and sequence files to create final bed file
    # Merge (combineSeqPrimerPos) takes about an hour to run
    ssh kolossus
    cd /cluster/data/hg17/bed/sts
    
    /cluster/bin/scripts/combineSeqPrimerPos stsMarkers.final primers/primers.final
        # creates *_pos.rdb
    /cluster/bin/scripts/createSTSbed /cluster/data/ncbi/sts.9/stsInfo2.bed \
                stsMarkers_pos.rdb > stsMap.bed

    # Set up sequence files
    ssh hgwdev
    mkdir -p /gbdb/hg17/sts.9/
    ln -s /cluster/data/ncbi/sts.9/all.STS.fa /gbdb/hg17/sts.9/all.STS.fa
    ln -s /cluster/data/ncbi/sts.9/all.primers.fa \
        /gbdb/hg17/sts.9/all.primers.fa

    # Load all files
    cd /cluster/data/hg17/bed/sts
    hgLoadSeq hg17 /gbdb/hg17/sts.9/all.STS.fa /gbdb/hg17/sts.9/all.primers.fa
    hgsql hg17 < ~kent/src/hg/lib/stsInfo2.sql
    hgsql hg17 < ~kent/src/hg/lib/stsAlias.sql
    cp /cluster/data/ncbi/sts.9/{stsInfo2.bed,stsAlias.bed} .
    hgsql hg17 -e 'load data local infile "stsInfo2.bed" into table stsInfo2'
    hgsql hg17 -e 'load data local infile "stsAlias.bed" into table stsAlias'
    hgLoadBed -noBin -tab -sqlTable=/cluster/home/kent/src/hg/lib/stsMap.sql \
	hg17 stsMap stsMap.bed
    hgLoadPsl -nobin -table=all_sts_primer hg17 primers/primers.psl
    hgLoadPsl -nobin -table=all_sts_seq hg17 stsMarkers.lifted.psl

    # update of information for D21S167 and D21S168 (2005-02-11, hartera)
    # currently X52289 associated with D21S168 
    # and X53367 associated with D21S167 - these need to be switched as they
    # are causing incorrect positioning
    # On Terry's advice, 
    # first manually update the accession field stsInfo2.bed so that the 
    # corrected version is carried through to the next version
    cd /cluster/data/hg17/bed/sts
    # manually change accessions in this file so now X52289 is associated
    # with D21S167 and X53367 is now associated with D21S168 
    # manually update the chromStart and chromEnd fields for these
    # records in stsMap.bed
    # this change was not carried through after filtering to change stsMap.bed
    # again and reload this table (DONE, 2005-02-18, hartera)
    chr21   39867340        39867513        D21S167 1000    7888    AF064860
    # becomes
    chr21   37117635        37117858        D21S167 1000    7888    AF064860

    chr21   37117635        37117858        D21S168 1000    103256  AP000699
    # becomes
    chr21   39867340        39867513        D21S168 1000    103256  AP000699
    
    # then reload the stsMap.bed and stsInfo2.bed files
    # copy this updated bed file back to ncbi directory
    cp stsInfo2.bed /cluster/data/ncbi/sts.9/
    # delete previous data before reloading tables
    hgsql hg17 -e 'delete from stsInfo2'
    hgsql hg17 -e 'drop table stsMap'
    hgsql hg17 -e 'load data local infile "stsInfo2.bed" into table stsInfo2'
    hgLoadBed -noBin -tab -sqlTable=/cluster/home/kent/src/hg/lib/stsMap.sql \
	hg17 stsMap stsMap.bed
    # (2005-02-19, hartera)
    # also need to update the psl alignment file and reload into all_sts_seq
    # for D21S168, the id is 103256, this is qName in the psl file
    # for D21S167, the id is 7888
    cd /cluster/data/hg17/bed/sts
    # manually update the stsMarkers.lifted.psl file with the new
    # co-ordinates as above.
    # (2005-02-23) Correct alignments.
    # need to swap the names for the alignments not just the start and end
    # coords as before as now the rest of the alignment data fields in the 
    # table are incorrect. Change the start and end co-ordinates and just swap 
    # the names for D21S167 and D21S168 in the psl file then reload the table. 
    # sort on the ID field (qName)
    sort -k 10n stsMarkers.lifted.psl > sts.lifted.sort
    mv sts.lifted.sort stsMarkers.lifted.psl 
    hgsql hg17 -e 'drop table all_sts_seq'
    hgLoadPsl -nobin -table=all_sts_seq hg17 stsMarkers.lifted.psl
    
    # Add new information after filtering the noOoc files 
    # (DONE, 2005-02-17, hartera)
    # latest psl file: stsMarker.lifted.new.psl is in fix dir    
    # Merge primer and sequence files to create final bed file
    ssh kolossus
    cd /cluster/data/hg17/bed/sts/fix
    nice /cluster/bin/scripts/combineSeqPrimerPos stsMarkers.final.new \
                                             ../primers/primers.final
    # creates *_pos.rdb
    /cluster/bin/scripts/createSTSbed /cluster/data/ncbi/sts.9/stsInfo2.bed \
                stsMarkers_pos.rdb > stsMap.bed
    awk '{print $6;}' stsMap.bed | sort -n | uniq > stsMap.ids
    diff stsMap.ids filt1000.ids 
    # There is only 1 id that does not make it into this set (109375)
    # There are 38 of the IDs to remove that do not appear in stsMap.ids
    # there are 65 therefore that appear in stsMap.bed: noOoctoremoveinStsMap
    foreach i (`cat noOoctoremoveinStsMap`) 
       awk 'BEGIN {OFS = "\t"} {if ($10 == "'$i'" && $8 >= 1000) \
        print $14, $16, $17, $10;}' stsMarkers.noOoc.lifted.filtToRemove.psl \
        >> stsMap.noOoc.toRemove.coords
    end 
    sort stsMap.noOoc.toRemove.coords > stsMap.noOoc.toRemove.coords.sort
    wc -l stsMap.noOoc.toRemove.coords.sort
    # 122
    # get the equivalent co-ordinates from stsMap.bed
    foreach i (`cat noOoctoremoveinStsMap`)
       awk 'BEGIN {OFS = "\t"} {if ($6 == "'$i'") print $1,$2,$3,$6;}' \
           stsMap.bed >> stsMap.toRemove.coords
    end
    sort stsMap.toRemove.coords > stsMap.toRemove.coords.sort
    wc -l stsMap.toRemove.coords.sort
    # 68
    diff stsMap.noOoc.toRemove.coords stsMap.toRemove.coords.sort
    # They are different co-ordinates in each set although the same ID
    # is represented.
    # none of the noOoc alignments are in stsMarkers.lifted.psl so add
    cp ../stsMarkers.lifted.psl stsMarkers.lifted.psl
    awk '{print $10}' stsMarkers.lifted.psl | sort -n | uniq > sts.liftedpsl.ids
    # none of the noOoc alignments are in stsMarkers.lifted.psl so add
    # the filtered version
    cp stsMarkers.lifted.psl stsMarkers.lifted.new.psl
    cat stsMarkers.noOoc.lifted.filt1000.psl >> stsMarkers.lifted.new.psl
    wc -l stsMarkers.lifted.new.psl
    # 91890 
    awk '{print $1;}' ../stsInfo2.bed | sort -n | uniq > stsInfo2.ids
    # diff with filt1000.ids and noOoc.IdsToRemove.txt
    # all of these are in stsInfo2.bed
    # need to remove info for the filtered out set but only for the 38 that
    # were removed from stsMap.bed - noOocnotinstsMap 
    perl removeById.pl noOocnotinstsMap stsInfo2.bed > stsInfo2.new.bed
cat << '_EOF_' > removeById.pl
#!/usr/bin/perl -w
use strict;

my $ids = $ARGV[0];
my $file = $ARGV[1];
# list of to remove
open(IDS, $ids) || die "Can not open $ids: $!\n";
# file of stsMarkers.final
open(FILE, $file) || die "Can not open $file: $!\n";
open(OUT, ">removedIds.txt") || die "Can not create removedIds.txt: $!\n";

my %idsHash;
while (<IDS>) {
   chomp;
   my @a = split(/\t/);
   my $id = $a[0];
   $idsHash{$id} = 1;
}
close IDS;

while (<FILE>) {
   my $l = $_;
   my $found = "FALSE";
   my @f = split(/\t/, $l);
   foreach my $k (keys(%idsHash)) {
      # if the id is contained in the key
      if ($k eq $f[0]) {
         $found = "TRUE";
         print OUT "$f[0]\n";
      }
   }
   if ($found eq "FALSE") {
      print $l;
   }
}
'_EOF_'
    # << emacs
    chmod +x removeById.pl
    # this removed data for all 38 of these Ids from stsInfo2.bed
    
    # need to reload database tables (2005-02-18, hartera)
    ssh hgwdev
    cd /cluster/data/hg17/bed/sts/fix
    hgsql hg17 -e 'drop table stsMap'
    hgsql hg17 -e 'drop table all_sts_seq'
    hgsql hg17 -e 'drop table stsInfo2'
    mv stsInfo2.new.bed stsInfo2.bed
    cp stsInfo2.bed /cluster/data/ncbi/sts.9/stsInfo2.bed
    mv stsMap.new.bed stsMap.bed
    mv stsMarkers.lifted.new.psl stsMarkers.lifted.psl
    hgLoadBed -noBin -tab -sqlTable=/cluster/home/kent/src/hg/lib/stsMap.sql \
	hg17 stsMap stsMap.bed
    hgLoadPsl -nobin -table=all_sts_seq hg17 stsMarkers.lifted.psl
    hgsql hg17 < ~kent/src/hg/lib/stsInfo2.sql
    hgsql hg17 -e 'load data local infile "stsInfo2.bed" into table stsInfo2'
    cd ..
    mkdir old
    mv stsMap.bed stsInfo2.bed stsMarkers.lifted.psl ./old
    mv ./fix/stsMap.bed ./fix/stsInfo2.bed ./fix/stsMarkers.lifted.psl .
   
    # Update of stsAlias table (DONE, 2005-02-24, hartera)
    # stsAlias filtered IDs removed
    # should have same IDs as in stsInfo2
    ssh eieio
    cd /cluster/data/hg17/bed/sts/fix
    awk '{print $2;}' ../stsAlias.bed | sort -n | uniq > alias.ids
    # 145985 alias.ids
    awk '{print $6;}' ../stsMap.bed | sort -n | uniq > stsMap.new.ids.sort
    awk '{print $1;}' ../stsInfo2.bed | sort -n | uniq > stsInfo.new.ids.sort
    # 16678 ids in stsInfo2 that are not in stsMap
    # 16717 ids in stsAlias that are not in stsMap
    # 38 ids in stsAlias that are not in stsInfo2
    cat stsMap.new.ids.sort stsInfo.new.ids.sort | sort -n | uniq \
        > stsMapandInfo.ids.sort
    diff stsMapandInfo.ids.sort alias.ids | grep '>' > idstoremoveAlias
    # there are 38 of these IDs to remove
    perl -pi.bak -e 's/> //' idstoremoveAlias
    cp ../stsAlias.bed .
    foreach i (`cat idstoremoveAlias`)
      awk '{if ($2 != "'$i'") print;}' stsAlias.bed > stsAlias.tmp
      mv stsAlias.tmp stsAlias.bed
    end
    # check that ids are removed from file and that they are the correct ones
    # all looks good
    cd /cluster/data/hg17/bed/sts
    # save old stsAlias file and copy new one to sts dir and to ncbi sts dir
    mv stsAlias.bed ./old
    cp ./fix/stsAlias.bed .
    cp stsAlias.bed /cluster/data/ncbi/sts.9/stsAlias.bed
    ssh hgwdev
    # remove old table data and reload
    hgsql hg17 -e 'delete from stsAlias'
    hgsql hg17 -e 'load data local infile "stsAlias.bed" into table stsAlias'

# PRUNE stsMap RECORDS (DONE 3/3/06)

  hgsql hg17 -e 'delete from stsMap where chromEnd-chromStart > 5000'
  
# RECOMBINATION RATES (2004-07-13 Terry)
#                       (2004-07-21 kate)

# The STS MArkers track must be completed prior to creating this track

    ssh eieio
    cd /cluster/data/hg17/bed
    mv recombRate recombRate.terry
    mkdir -p recombRate
	cd recombRate

# Copy other necessary files here (in future, can take from previous version)
# NOTE: these are stable, and could be saved in a permanent spot
    cp /projects/hg2/booch/psl/info/decode_all .
    cp /projects/hg2/booch/psl/info/marshfield_all .
    cp /projects/hg2/booch/psl/info/genethon_all .
	
# Determine maximum concordant set of markers for each of the maps
    /cluster/bin/scripts/assignGPsts -full -maxcon \
        /cluster/data/ncbi/sts.9/stsAlias.bed \
        /cluster/data/hg17/bed/sts/stsMarkers_pos.rdb \
        decode_all > decode.marker.rdb
    /cluster/bin/scripts/assignGPsts -full -maxcon \
        /cluster/data/ncbi/sts.9/stsAlias.bed \
        /cluster/data/hg17/bed/sts/stsMarkers_pos.rdb \
        marshfield_all > marshfield.marker.rdb
    /cluster/bin/scripts/assignGPsts -full -maxcon \
        /cluster/data/ncbi/sts.9/stsAlias.bed \
        /cluster/data/hg17/bed/sts/stsMarkers_pos.rdb \
        genethon_all > genethon.marker.rdb

# Determine the rates for each of the maps
    /cluster/bin/scripts/markers_to_recomb_rate.terry.pl decode.marker.rdb \
            /cluster/data/hg17/chrom.sizes 1000000 1000000 \
                > decode_1mb_slide_1mb
    /cluster/bin/scripts/markers_to_recomb_rate.terry.pl genethon.marker.rdb \
            /cluster/data/hg17/chrom.sizes 1000000 1000000 \
                > genethon_1mb_slide_1mb
        # Marker number 2 at position 120005974 on chr9 is out of genetic distance order. DISCARDING
    /cluster/bin/scripts/markers_to_recomb_rate.terry.pl marshfield.marker.rdb \
            /cluster/data/hg17/chrom.sizes 1000000 1000000 \
                > marshfield_1mb_slide_1mb
        # Marker number 1 at position 124276104 on chr9 is out of genetic distance order. DISCARDING

# Convert files to proper format
    /cluster/bin/scripts/convertRecombRate decode_1mb_slide_1mb \
        /cluster/data/hg17/inserts \
        /cluster/data/hg17 1000 > decode_1mb_slide_1mb_conv
    /cluster/bin/scripts/convertRecombRate marshfield_1mb_slide_1mb \
        /cluster/data/hg17/inserts \
         /cluster/data/hg17 1000 > marshfield_1mb_slide_1mb_conv
    /cluster/bin/scripts/convertRecombRate genethon_1mb_slide_1mb \
        /cluster/data/hg17/inserts \
	    /cluster/data/hg17 1000 > genethon_1mb_slide_1mb_conv

# Create bed file and load
    /cluster/bin/scripts/createRRbed decode_1mb_slide_1mb_conv \
        marshfield_1mb_slide_1mb_conv genethon_1mb_slide_1mb_conv \
                > recombRate.bed
    hgLoadBed -noBin -tab \
        -sqlTable=/cluster/home/kent/src/hg/lib/recombRate.sql \
	    hg17 recombRate recombRate.bed
	   

# FISH CLONES (DONE 2004-07-22 Kate)
#  Reloaded 2004-09-36 after Terry Furey reworked fishClones.c
#       to improve scoring

# The STS Marker, Coverage, and BAC End Pairs tracks must be completed prior to 
# creating this track

    ssh eieio
    mkdir -p /cluster/data/ncbi/fishClones/fishClones.2004-07/
    cd /cluster/data/ncbi/fishClones/fishClones.2004-07/

# Download information from NCBI
        # point browser at http://www.ncbi.nlm.nih.gov/genome/cyto/cytobac.cgi?CHR=all&VERBOSE=ctg
        # change "Show details on sequence-tag" to "yes"
        # change "Download or Display" to "Download table for UNIX"
        # press Submit - save as /cluster/data/ncbi/fishClones/fishClones.2004-07/hbrc.txt
    chmod 664 /cluster/data/ncbi/fishClones/fishClones.2004-07/hbrc.txt

# Get current clone/accession information
    wget http://www.ncbi.nlm.nih.gov/genome/clone/DATA/clac.out

# Create initial Fish Clones bed file
    mkdir -p /cluster/data/hg17/bed/fishClones
    cd /cluster/data/hg17/bed/fishClones

# Copy previous sts info from fhcrc (take from previous build in future)
    cp ~booch/tracks/fish/fhcrc.sts .
    fishClones -verbose=1 -fhcrc=fhcrc.sts -noBin hg17 \
         /cluster/data/ncbi/fishClones/fishClones.2004-07/hbrc.txt \
         /cluster/data/ncbi/fishClones/fishClones.2004-07/clac.out \
         /cluster/data/ncbi/bacends/human/bacends.4/cl_acc_gi_len \
         /cluster/data/hg17/bed/bacends/lifted/bacEnds.lifted.psl \
            fishClones_initial
	     
# Get sequences for accessions not in genome
    ssh eieio
    mkdir -p /cluster/bluearc/hg17/fishClones/
    cd /cluster/bluearc/hg17/fishClones/
	# goto http://www.ncbi.nlm.nih.gov/entrez/batchentrez.cgi?db=Nucleotide
	# select file "/cluster/data/hg17/bed/fishClones/fishClones_initial.acc"
	# change output to FASTA format
	# download results to "/cluster/bluearc/hg17/fishClones/notFound.fa"

# Align these using blat
    cp ~booch/tracks/gs.17/build34/fish/convert.pl .
    cp ~booch/tracks/gs.17/build34/fish/blatAll.pl .
        # edited to use ooc file on bluearc, so can run on kolossus
    convert.pl  < notFound.fa > notFound.convert.fa
    mkdir out
    blatAll.pl /cluster/data/hg17 notFound.convert.fa out
        # creates raw.psl, not.found.psl

# Make final fishClones file with this new clone placement info
    cd /cluster/data/hg17/bed/fishClones
    fishClones -verbose=1 -fhcrc=fhcrc.sts -noBin \
         -psl=/cluster/bluearc/hg17/fishClones/not.found.psl hg17 \
         /cluster/data/ncbi/fishClones/fishClones.2004-07/hbrc.txt \
         /cluster/data/ncbi/fishClones/fishClones.2004-07/clac.out \
         /cluster/data/ncbi/bacends/human/bacends.4/cl_acc_gi_len \
         /cluster/data/hg17/bed/bacends/lifted/bacEnds.lifted.psl fishClones

# Load the track
    ssh hgwdev
    cd /cluster/data/hg17/bed/fishClones
    hgLoadBed -noBin -tab \
        -sqlTable=/cluster/home/kent/src/hg/lib/fishClones.sql \
	hg17 fishClones fishClones.bed
            # Loaded 10601 elements of size 16

# fixed bad table entry (2004-08-12 kate)
# NOTE: this won't be necessary in the future, as the fishClones program
# will now accomodate more bad input data.
    hgsql hg17 -e "update fishClones set bandEnds='1q43,Yp' where  name='RP11-188A4' and placeCount=2"


# CHROMOSOME BANDS TRACK (2004-07-13 Terry)

# This must wait until the Fish Clones tracks is done
    mkdir -p /cluster/data/hg17/bed/cytoband
    cd /cluster/data/hg17/bed/cytoband

# Copy in some necessary files (usually from previous version)
    cp /projects/hg2/booch/psl/cytobands/pctSetBands.txt .
    cp /projects/hg2/booch/psl/cytobands/ISCN800.txt .

# Create some preliminary information files
    /cluster/bin/scripts/createSetBands pctSetBands.txt \
	/cluster/data/hg17/inserts /cluster/data/hg17  100 > setBands.txt
    /cluster/bin/scripts/makeBands ISCN800.txt /cluster/data/hg17 > cytobands.pct.bed
    /cluster/bin/scripts/makeBandRanges cytobands.pct.bed > cytobands.pct.ranges

# Reformat fishClones file
    /cluster/bin/scripts/createBanderMarkers \
	/cluster/data/hg17/bed/fishClones/fishClones.bed > fishClones.txt

# Create bed file
    /cluster/bin/scripts/runBander fishClones.txt \
	ISCN800.txt setBands.txt /cluster/data/hg17

    # Should be 862 bands
    wc cytobands.bed
    # 862    4310   30748 cytobands.bed

# Load track
    hgLoadBed -noBin -tab -sqlTable=/cluster/home/kent/src/hg/lib/cytoBand.sql \
	hg17 cytoBand cytobands.bed
	    
# Load ideogram table
    hgLoadBed -noBin -tab -sqlTable=/cluster/home/booch/src/hg/lib/cytoBandIdeo.sql \
	hg17 cytoBandIdeo cytobands.bed


# CHROMOSOME BANDS TRACK REDO (2004-07-22 Kate)
#       Just to make sure we know the proper steps.
#       The tables were not reloaded, as Terry has already
#       sent the data to NCBI

# This must wait until the Fish Clones tracks is done
    ssh kolossus
    mkdir -p /cluster/data/hg17/bed/cytoband.kate
    cd /cluster/data/hg17/bed/cytoband.kate

# Copy in some necessary files (usually from previous version)
    cp /projects/hg2/booch/psl/cytobands/pctSetBands.txt .
    cp /projects/hg2/booch/psl/cytobands/ISCN800.txt .

# Create some preliminary information files
    /cluster/bin/scripts/createSetBands pctSetBands.txt \
	/cluster/data/hg17/inserts /cluster/data/hg17  100 > setBands.txt
    /cluster/bin/scripts/makeBands ISCN800.txt \
        /cluster/data/hg17 > cytobands.pct.bed
    /cluster/bin/scripts/makeBandRanges cytobands.pct.bed \
        > cytobands.pct.ranges

# Reformat fishClones file
    /cluster/bin/scripts/createBanderMarkers \
	/cluster/data/hg17/bed/fishClones/fishClones.bed > fishClones.txt

# Create bed file
    ssh eieio
    cd /cluster/data/hg17/bed/cytoband.kate
    /cluster/bin/scripts/runBander fishClones.txt \
	ISCN800.txt setBands.txt /cluster/data/hg17
    # NOTE: fails on kolossus (C++ compiler different ??)

    # Should be 862 bands
    wc  -l cytobands.bed
    # 862    cytobands.bed

# NOTE - don't load tracks, as Terry has already sent his
#       versions to NCBI

# Load track
    #hgLoadBed -noBin -tab \
        # -sqlTable=/cluster/home/kent/src/hg/lib/cytoBand.sql \
	# hg17 cytoBand cytobands.bed
	    
# Load ideogram table
    #hgLoadBed -noBin -tab \
        # -sqlTable=/cluster/home/booch/src/hg/lib/cytoBandIdeo.sql \
	# hg17 cytoBandIdeo cytobands.bed


# LOAD AFFYRATIO (DONE - 2004-07-14 - Hiram)
#	Copied from Hg16 doc
    # Set up cluster job to align consenesus/exemplars to hg17
    ssh eieio
    mkdir /cluster/bluearc/hg17/affyGnf
    cp -p /projects/compbio/data/microarray/affyGnf/sequences/HG-U95/HG-U95Av2_all.fa /cluster/bluearc/hg17/affyGnf

    ssh kkr1u00
    mkdir -p /iscratch/i/affyGnf
    cp -p /cluster/bluearc/hg17/affyGnf/* /iscratch/i/affyGnf
    /cluster/bin/iSync

    ssh kki
    mkdir /cluster/data/hg17/bed/affyGnf.2004-06-09
    cd /cluster/data/hg17/bed/affyGnf.2004-06-09
    ls -1 /iscratch/i/affyGnf/* > affy.lst
    ls -1 /iscratch/i/gs.18/build35/maskedContigs/* > allctg.lst
    cat << '_EOF_' > template.sub
#LOOP
/cluster/bin/i386/blat -fine -mask=lower -minIdentity=95 -ooc=/iscratch/i/gs.18/build35/hg17.11.ooc  $(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    gensub2 allctg.lst affy.lst template.sub jobList
    mkdir psl
    para create jobList
# Completed: 380 of 380 jobs
# CPU time in finished jobs:       2922s      48.70m     0.81h    0.03d  0.000 y
# IO & Wait Time:                  1146s      19.10m     0.32h    0.01d  0.000 y
# Average job time:                  11s       0.18m     0.00h    0.00d
# Longest job:                       80s       1.33m     0.02h    0.00d
# Submission to last job:           333s       5.55m     0.09h    0.00d

    # Do sort, best in genome filter, and convert to chromosome coordinates
    # to create affyU95.psl
    ssh eieio
    cd /cluster/data/hg17/bed/affyGnf.2004-06-09
    pslSort dirs raw.psl tmp psl

    # change filter parameters for these sequences. only use alignments that
    # cover 30% of sequence and have at least 95% identity in aligned
    # region. 
    # minAli = 0.97 too high. low minCover as a lot of n's in these
    # sequences
    pslReps -minCover=0.3 -sizeMatters -minAli=0.95 -nearTop=0.005 \
	raw.psl contig.psl /dev/null
    liftUp affyU95.psl ../../jkStuff/liftAll.lft warn contig.psl
    #   Eliminate the long names
    sed -e "s/U95Av2://" affyU95.psl | sed -e "s/;//" > affyU95shortQname.psl

    # Merge with spot data and load into database. added -chip flag to 
    # affyPslAndAtlasToBed to allow correct parsing
    ssh hgwdev
    cd /cluster/data/hg17/bed/affyGnf.2004-06-09
    
    /cluster/home/sugnet/bin/i386/affyPslAndAtlasToBed -chip=U95Av2 \
	affyU95shortQname.psl \
	/projects/compbiodata/microarray/affyGnf/human_atlas_U95_gnf.noquotes.txt \
	affyRatio.bed affyRatio.exr > affyPslAndAtlasToBed.log 2>&1

    hgLoadBed -sqlTable=$HOME/src/hg/lib/affyRatio.sql hg17 \
	affyRatio affyRatio.bed
    #	Loaded 12740 elements of size 15

    mkdir affyU95
    hgLoadPsl hg17 -table=affyU95 affyU95shortQname.psl
    # sequences loaded 2004-08-06
    hgLoadSeq -abbr=U95Av2: hg17 /gbdb/hgFixed/affyProbes/HG-U95Av2_all.fa
    #	Advisory lock created
    #	Creating .tab file
    #	Adding /gbdb/hgFixed/affyProbes/HG-U95Av2_all.fa
    #	12386 sequences
    #	Updating seq table
    #	Advisory lock has been released
    #	All done

# Load AFFYUCLANORM, extended version of affyUcla track. Hopefully
# final freeze of data set.		(DONE - 2004-07-14 - Hiram)
    ssh kk
    mkdir /cluster/data/hg17/bed/affyUclaNorm
    cd /cluster/data/hg17/bed/affyUclaNorm

    cp /projects/compbio/data/microarray/affyUcla/sequences/HG-U133AB_all.fa .
    ls -1 /scratch/hg/gs.18/build35/maskedContigs/* > contig.lst

    cat << '_EOF_' > gsub
#LOOP
/cluster/bin/i386/blat -fine -mask=lower -minIdentity=95 -ooc=/scratch/hg/h/11.ooc  $(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

    mkdir psl
    ls HG-U133AB_all.fa > affy.lst
    gensub2 contig.lst affy.lst gsub jobList
    para create jobList
    para try
    para check
    para push ... etc
# Completed: 380 of 380 jobs
# CPU time in finished jobs:      20070s     334.51m     5.58h    0.23d  0.001 y
# IO & Wait Time:                162784s    2713.06m    45.22h    1.88d  0.005 y
# Average job time:                 481s       8.02m     0.13h    0.01d
# Longest job:                      735s      12.25m     0.20h    0.01d
# Submission to last job:           771s      12.85m     0.21h    0.01d

    ssh eieio
    cd /cluster/data/hg17/bed/affyUclaNorm
    pslSort dirs hg17.affyU133AB_all.psl tmp psl
    wc hg17.affyU133AB_all.psl
    #	61022 1281401 12934919 hg17.affyU133AB_all.psl
    liftUp hg17.affyU133AB_all.lifted.psl \
	/cluster/data/hg17/jkStuff/liftAll.lft warn hg17.affyU133AB_all.psl 
    pslReps -minCover=0.5 -sizeMatters -minAli=0.97 \
	-nearTop=0.005  hg17.affyU133AB_all.lifted.psl \
	hg17.affyU133AB_all.lifted.pslReps.psl out.psr
    #	Processed 61017 alignments
    affyUclaMergePslData -pslFile=hg17.affyU133AB_all.lifted.pslReps.psl \
	-affyFile=/projects/compbio/data/microarray/affyUcla/data/feature_biomaterial_chip_logratios_formatForTrack.txt \
	-bedOut=hg17.affyUcla.bed \
	-expRecordOut=hg17.affyUcla.expRecords \
	-expFile=/projects/compbio/data/microarray/affyUcla/data/expNames.sorted.txt

    ~/kent/src/hg/affyGnf/addUclaAnnotations.pl hg17.affyUcla.expRecords \
	/projects/compbio/data/microarray/affyUcla/data/normal_tissue_database_annotations2.txt > hg17.affyUcla.annotations.expRecords

    # Load the databases
    ssh hgwdev
    cd /cluster/data/hg17/bed/affyUclaNorm
    sed -e 's/affyRatio/affyUclaNorm/' ~/kent/src/hg/lib/affyRatio.sql \
	> affyUclaNorm.sql
    hgLoadBed hg17 affyUclaNorm hg17.affyUcla.bed -sqlTable=affyUclaNorm.sql

# MAKE AFFY U133 - made after above affyUclaNorm (DONE - 2004-07-15 - Hiram)
    #	Someday the names can be fixed.
    ssh hgwdev
    mkdir /cluster/data/hg17/bed/affyU133
    cd /cluster/data/hg17/bed/affyU133
    ln -s ../affyUclaNorm/hg17.affyU133AB_all.lifted.pslReps.psl affyU133.psl
    hgLoadPsl hg17 affyU133.psl
    #	hgsql -e "select count(*) from affyU133;" hg17
    #	row count in hg16: 45693, in hg17: 44620
    hgLoadSeq hg17 /gbdb/hgFixed/affyProbes/HG-U133AB_all.fa
    #	44792 sequences

# MAKE LINEAGE-SPECIFIC REPEATS FOR CHICKEN & FUGU (DONE 2004-06-10 kate)
    # In an email 2/13/04 to Angie, Arian said we could treat all 
    # human repeats as 
    # lineage-specific for human-chicken blastz.  
    # and Angie did the same for fugu.
    # Lacking input from Arian, and using blastzSelf as a model,
    # I'm also using all human repeats for the human/chimp blastz.
    # Scripts expect *.out.spec filenames.
    ssh kkr1u00
    cd /cluster/data/hg17
    mkdir /iscratch/i/hg17/linSpecRep.chicken
    foreach f (/iscratch/i/hg17/rmsk/chr*.fa.out)
      cp -p $f /iscratch/i/hg17/linSpecRep.chicken/$f:t:r:r.out.spec
    end
    ln -s /iscratch/i/hg17/linSpecRep.chicken \
          /iscratch/i/hg17/linSpecRep.fugu
    ln -s /iscratch/i/hg17/linSpecRep.chicken \
          /iscratch/i/hg17/linSpecRep.chimp
    iSync


# BLASTZ FUGU (FR1) (DONE 2004-06-24 kate)
    ssh kk
    mkdir -p /cluster/data/hg17/bed/blastz.fr1.2004-06-10
    ln -s /cluster/data/hg17/bed/blastz.fr1.2004-06-10 \
            /cluster/data/hg17/bed/blastz.fr1
    cd /cluster/data/hg17/bed/blastz.fr1
    # Set L=6000 (more relaxed than chicken) and abridge repeats.
    # Treat all repeats as lineage-specific (reuse linSpecRep.Chicken).
    cat << '_EOF_' > DEF
# human vs. fugu
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin

ALIGN=blastz-run
BLASTZ=blastz

# Reuse parameters from human-chicken.
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_Q=/cluster/data/blastz/HoxD55.q
BLASTZ_ABRIDGE_REPEATS=1

# TARGET: Human
SEQ1_DIR=/iscratch/i/hg17/bothMaskedNibs
SEQ1_RMSK=
SEQ1_FLAG=
SEQ1_SMSK=/iscratch/i/hg17/linSpecRep.fugu
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Fugu
SEQ2_DIR=/iscratch/i/fr1/nib
SEQ2_RMSK=
SEQ2_FLAG=
SEQ2_SMSK=/iscratch/i/fr1/linSpecRep
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/blastz.fr1

DEF=$BASE/DEF
RAW=$BASE/raw
CDBDIR=$BASE
SEQ1_LEN=$BASE/S1.len
SEQ2_LEN=$BASE/S2.len
'_EOF_'
    # << this line keeps emacs coloring happy

    bash # if a csh/tcsh user
    source DEF
    mkdir $RAW run.0
    /cluster/home/angie/hummus/make-joblist $DEF > $BASE/run.0/j
    # GOT HERE
    sh ./xdir.sh
    cd run.0
    sed -e 's@^blastz-run@/cluster/bin/penn/blastz-run@' j > jobList
    para create jobList
        # 11935 jobs
    para try
    para check
    para push
# Completed: 11935 of 11935 jobs
# CPU time in finished jobs:    4673316s   77888.60m  1298.14h   54.09d  0.148 y
# IO & Wait Time:                329249s    5487.48m    91.46h    3.81d  0.010 y
# Average job time:                 419s       6.99m     0.12h    0.00d
# Longest job:                      714s      11.90m     0.20h    0.01d
# Submission to last job:          5575s      92.92m     1.55h    0.06d

    # second cluster run: lift raw alignments -> lav dir
    ssh kki
    cd /cluster/data/hg17/bed/blastz.fr1
    bash # if a csh/tcsh user
    source DEF
    mkdir run.1 lav
    /cluster/bin/scripts/blastz-make-out2lav $DEF $BASE > $BASE/run.1/jobList
    cd run.1
    wc -l jobList
    para create jobList
        # 341 jobs
    para try
    para check 
    para push
# CPU time in finished jobs:        315s       5.26m     0.09h    0.00d  0.000 y
# IO & Wait Time:                  4451s      74.18m     1.24h    0.05d  0.000 y
# Average job time:                  14s       0.23m     0.00h    0.00d
# Longest job:                      107s       1.78m     0.03h    0.00d
# Submission to last job:           368s       6.13m     0.10h    0.00d

    # third run: lav -> axt
    ssh kki
    cd /cluster/data/hg17/bed/blastz.fr1
    mkdir axtChrom pslChrom run.2
    cd run.2
    cat << 'EOF' > do.csh
#!/bin/csh -ef
cd $1
set chr = $1:t
cat `ls -1 *.lav | sort -g` \
| lavToAxt stdin \
        /iscratch/i/hg17/bothMaskedNibs /iscratch/i/fr1/nib stdout \
| axtSort stdin ../../axtChrom/$chr.axt 
axtToPsl ../../axtChrom/$chr.axt ../../S1.len ../../S2.len \
        ../../pslChrom/$chr.psl
'EOF'
    # << this line keeps emacs coloring happy
    chmod a+x do.csh
    cp /dev/null jobList
    foreach d (../lav/chr*)
      echo "do.csh $d" >> jobList
    end
    para create jobList
        # 41 jobs
    para try
    para check
    para push


# CHAIN FUGU BLASTZ (2004-06-11 kate)
    # Run axtChain on little cluster
    ssh kki
    cd /cluster/data/hg17/bed/blastz.fr1
    mkdir -p axtChain/run1
    cd axtChain/run1
    mkdir out chain
    ls -1S /cluster/data/hg17/bed/blastz.fr1/axtChrom/*.axt \
      > input.lst
    cat << '_EOF_' > gsub
#LOOP
doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    # Reuse gap penalties from chicken run.
    cat << '_EOF_' > temp.gap
tablesize	11
smallSize	111
position	1	2	3	11	111	2111	12111	32111	72111	152111	252111
qGap	325	360	400	450	600	1100	3600	7600	15600	31600	56600
tGap	325	360	400	450	600	1100	3600	7600	15600	31600	56600
bothGap	625	660	700	750	900	1400	4000	8000	16000	32000	57000
'_EOF_'
    # << this line makes emacs coloring happy
    sed 's/  */\t/g' temp.gap > ../../fuguHumanTuned.gap
    rm -f temp.gap

    cat << '_EOF_' > doChain
#!/bin/csh
axtChain -scoreScheme=/cluster/data/blastz/HoxD55.q \
                      -linearGap=../../fuguHumanTuned.gap \
                      -minScore=5000 $1 \
    /iscratch/i/hg17/bothMaskedNibs \
    /iscratch/i/fr1/nib $2 > $3
'_EOF_'
    # << this line makes emacs coloring happy
    chmod a+x doChain
    gensub2 input.lst single gsub jobList
    para create jobList
        # 46 jobs
    para try
    para check
    para push
        # 1 crashed job -- chr6_hla_hap1.chain is empty
# CPU time in finished jobs:        610s      10.16m     0.17h    0.01d  0.000 y
# IO & Wait Time:                  1644s      27.40m     0.46h    0.02d  0.000 y
# Average job time:                  50s       0.83m     0.01h    0.00d
# Longest job:                      233s       3.88m     0.06h    0.00d
# Submission to last job:           339s       5.65m     0.09h    0.00d

    # now on the cluster server, sort chains
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.fr1/axtChain
    chainMergeSort run1/chain/*.chain > all.chain
    chainSplit chain all.chain
    rm run1/chain/*.chain

    # Load chains into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.fr1/axtChain/chain
    foreach i (*.chain)
        set c = $i:r
        echo loading $c
        hgLoadChain hg17 ${c}_chainFr1 $i
    end
    featureBits hg16 chainFr1Link
        # 50709290 bases of 2865248791 (1.770%) in intersection


# ANCIENT REPEAT TABLE (2004-06-11 kate)

    # The netClass operations requires an "ancientRepeat" table in one 
    # of the databases.
    # This is a hand curated table obtained from Arian.

    ssh hgwdev
    mkdir -p /cluster/data/hg17/bed/ancientRepeat
    cd /cluster/data/hg17/bed/ancientRepeat
    # mysqldump needs write permission to this directory
    chmod 777 .
    hgsqldump --all --tab=. hg15 ancientRepeat
    chmod 775 .
    hgsql hg17 < ancientRepeat.sql
    echo "LOAD DATA LOCAL INFILE 'ancientRepeat.txt' into table ancientRepeat"\
                | hgsql hg17


# NET FUGU BLASTZ (2004-06-11 kate)
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.fr1/axtChain
    chainPreNet all.chain ../S1.len ../S2.len stdout \
    | chainNet stdin -minSpace=1 ../S1.len ../S2.len stdout /dev/null \
    | netSyntenic stdin noClass.net

    # Add classification info using db tables:
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.fr1/axtChain
    netClass noClass.net hg17 fr1 human.net

    # Make a 'syntenic' subset:
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.fr1/axtChain
    rm noClass.net
    # Make a 'syntenic' subset of these with
    netFilter -syn human.net > humanSyn.net

    # Load the nets into database 
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.fr1/axtChain
    netFilter -minGap=10 human.net |  hgLoadNet hg17 netFr1 stdin
    #netFilter -minGap=10 humanSyn.net | hgLoadNet hg17 netSyntenyFr1 stdin


# EXTRACT AXT'S AND MAF'S FROM THE NET (kate)
    # NOTE: Redo 2005-08-16 to fix overlap problem (use 8/05 netToAxt)
# Replace bad chr5 axtNet and mafNet (2006-01-05 kate) 
    ssh kkstore2
    cd /cluster/data/hg17/bed/blastz.fr1/axtChain
    netSplit human.net humanNet
    mkdir -p ../axtNet ../mafNet
cat > makeMaf.csh << 'EOF'
    foreach f (humanNet/chr*.net)
        set c = $f:t:r
        echo "axtNet on $c"
        netToAxt humanNet/$c.net chain/$c.chain /cluster/data/hg17/nib /cluster/data/fr1/nib stdout | axtSort stdin ../axtNet/$c.axt
        axtToMaf ../axtNet/$c.axt \
            /cluster/data/hg17/chrom.sizes /cluster/data/fr1/chrom.sizes \
            ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=fr1.
    end
'EOF'
    csh makeMaf.csh >&! makeMaf.log &
    tail -100f makeMaf.log
    mkdir -p /cluster/bluearc/hg17/mafNet
    cp -rp ../mafNet /cluster/bluearc/hg17/mafNet/fr1


# FUGU FR1 DOWNLOADS (DONE 2004-09-17 kate)
#    REDO axtNet downloads for fix, above (2005-09-12 kate)
# Replace bad chr5 axtNet and mafNet (2006-01-05 kate) 

    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.fr1/axtChain
    ln -s all.chain fugu.chain
    mkdir gz
    gzip -c fugu.chain > gz/fugu.chain.gz
    gzip -c human.net > gz/fugu.net.gz
    cd ../axtNet
    nice gzip *.axt

    ssh kkstore02
    cd /cluster/data/hg17/bed/blastz.fr1/axtNet
    gzip *.axt
    md5sum *.gz > md5sum.txt

    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17
    mkdir -p vsFr1
    cd vsFr1
    # Copy and edit README
    cp /cluster/data/hg17/bed/blastz.fr1/axtChain/gz/*.gz .
    md5sum *.gz > md5sum.txt

    mv axtNet axtNet.old
    ln -s /cluster/data/hg17/bed/blastz.fr1/axtNet .


# PRODUCE FUGU BLAT ALIGNMENT (DONE - 2004-07-07 - Hiram)

    # Use masked scaffolds from fr1 assembly (same sequence as
    # previous BlatFugu, however it's repeat and TRF-masked).

    ssh kk
    mkdir /cluster/data/hg17/bed/blatFr1
    cd /cluster/data/hg17/bed/blatFr1
    mkdir psl 
    # next time, use N?_?????? (to pick up NG_ contigs)
    foreach f ( `cat /cluster/data/hg17/contig.lst` )
      set c=$f:t:r
      echo $c
      mkdir psl/$c
    end

    # create cluster job
    mkdir run
    cd run
    ls -1S /iscratch/i/fugu/trfFa/*.fa > fugu.lst
    ls -1S /scratch/hg/gs.18/build35/maskedContigs/*.fa > human.lst
cat << 'EOF' > gsub
#LOOP
/cluster/bin/i386/blat -mask=lower -qMask=lower -q=dnax -t=dnax {check in line+ $(path1)} {check in line+ $(path2)} {check out line+ /cluster/data/hg17/bed/blatFr1/psl/$(root1)/$(root1)_$(root2).psl}
#ENDLOOP
'EOF'
    # << keep emacs happy
    gensub2 human.lst fugu.lst gsub jobList
    para create spec
       # 219640 jobs  
    para try
    para check
    para push -maxQueue=300000 -maxPush=220000
    para check
# Completed: 219640 of 219640 jobs
# CPU time in finished jobs:    5206945s   86782.41m  1446.37h   60.27d  0.165 y
# IO & Wait Time:                797791s   13296.52m   221.61h    9.23d  0.025 y
# Average job time:                  27s       0.46m     0.01h    0.00d
# Longest job:                      951s      15.85m     0.26h    0.01d
# Submission to last job:          7553s     125.88m     2.10h    0.09d
        # cd psl
        # count files with aligments
        # find . -not -size 427c | wc -l
        # 44558
        # count files with no aligments
        # find . -size 427c | wc -l
        # 175463

   # When cluster run is done, sort alignments
   # into chrom directory
    ssh eieio
    cd /cluster/data/hg17/bed/blatFr1
    pslCat -dir psl/N?_?????? | \
      liftUp -type=.psl stdout \
        /cluster/data/hg17/jkStuff/liftAll.lft warn stdin | \
      pslSortAcc nohead chrom temp stdin
        # 65 minutes ?
        # Processed 216595 lines into 1 temp files

    # Rename to correspond with tables and load into database:
    ssh hgwdev
    cd /cluster/data/hg17/bed/blatFr1/chrom
    foreach i (chr*.psl)
        set r = $i:r
        echo mv $i ${r}_blatFr1.psl
        mv $i ${r}_blatFr1.psl
    end

    # lift fugu scaffolds to Fugu browser chrUn,
    # so you can link to other browser.  And don't need to load sequence
    cd /cluster/data/hg17/bed/blatFr1
    liftUp -pslQ all.psl /cluster/data/fr1/fugu_v3.masked.lft warn chrom/*.psl

    hgLoadPsl -table=blatFr1 hg17 all.psl
    #	load of blatFr1 did not go as planned: 216595 record(s),
    #	0 row(s) skipped, 3 warning(s) loading psl.tab
    #	featureBits hg17 blatFr1 refGene:CDS
    #	13563544 bases of 2866216770 (0.473%) in intersection
    #	featureBits hg16 blatFr1 refGene:CDS
    #	13547219 bases of 2865248791 (0.473%) in intersection
    #	featureBits hg15 blatFugu refGene:CDS
    #	12427544 bases of 2866466359 (0.434%) in intersection

#  BLASTZ RAT RN3 (DONE - 2004-06-14 - Hiram)

    ssh kk
    mkdir -p /cluster/data/hg17/bed/blastz.rn3.2004-06-11
    cd /cluster/data/hg17/bed
    ln -s  blastz.rn3.2004-06-11 blastz.rn3
    cd blastz.rn3

    cat << '_EOF_' > DEF
# rat vs. human
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/home/angie/schwartzbin:/cluster/home/kent/bin/i386

ALIGN=blastz-run
BLASTZ=blastz
BLASTZ_H=2000
BLASTZ_ABRIDGE_REPEATS=1

# TARGET
# Human
SEQ1_DIR=/iscratch/i/gs.18/build35/bothMaskedNibs
# not used
SEQ1_RMSK=
# not used
SEQ1_FLAG=
SEQ1_SMSK=/iscratch/i/gs.18/build35/linSpecRep.notInRat
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY
# Rat
SEQ2_DIR=/iscratch/i/rn3/bothMaskedNibs
# not currently used
SEQ2_RMSK=
# not currently used
SEQ2_FLAG=
SEQ2_SMSK=/cluster/bluearc/rat/rn3/linSpecRep.notInHuman
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=30000000
SEQ2_LAP=0

BASE=/cluster/store5/gs.18/build35/bed/blastz.rn3

DEF=$BASE/DEF
RAW=$BASE/raw
CDBDIR=$BASE
SEQ1_LEN=$BASE/S1.len
SEQ2_LEN=$BASE/S2.len
'_EOF_'
    # << this line makes emacs coloring happy

    # prepare first cluster run
    ssh kk
    cd /cluster/data/hg17/bed/blastz.rn3
    source DEF
    # script copied over from /cluster/data/mm4/jkStuff/BlastZ_run0.sh
    #	it is a generic script and works for any assembly
    /cluster/data/hg17/jkStuff/BlastZ_run0.sh
    cd run.0
    para try, check, push, check, ....
Completed: 41943 of 41943 jobs
CPU time in finished jobs:   15330421s  255507.02m  4258.45h  177.44d  0.486 y
IO & Wait Time:                673809s   11230.15m   187.17h    7.80d  0.021 y
Average job time:                 382s       6.36m     0.11h    0.00d
Longest job:                     4651s      77.52m     1.29h    0.05d
Submission to last job:        169197s    2819.95m    47.00h    1.96d

    #	Second cluster run to convert the .out's to .lav's
    #	You do NOT want to run this on the big cluster.  It brings
    #	the file server to its knees.  Run this on the small cluster.
    ssh kki
    cd /cluster/data/hg17/bed/blastz.rn3
    # script copied over from /cluster/data/mm4/jkStuff/BlastZ_run1.sh
    #	fixup machine check, should be kki, not kk
    /cluster/data/hg17/jkStuff/BlastZ_run1.sh
    cd run.1
    para try, check, push, etc ...
# Completed: 341 of 341 jobs
# CPU time in finished jobs:       1894s      31.56m     0.53h    0.02d  0.000 y
# IO & Wait Time:                  6271s     104.52m     1.74h    0.07d  0.000 y
# Average job time:                  24s       0.40m     0.01h    0.00d
# Longest job:                      131s       2.18m     0.04h    0.00d
# Submission to last job:           590s       9.83m     0.16h    0.01d

    #	Third cluster run to convert lav's to axt's
    cd /cluster/data/hg17/bed/blastz.rn3
    #	The copy of this in mm4 was broken, fixed here
    /cluster/data/hg17/jkStuff/BlastZ_run2.sh
    cd run.2
    para try, check, push, etc ...
# Completed: 46 of 46 jobs
# CPU time in finished jobs:        426s       7.09m     0.12h    0.00d  0.000 y
# IO & Wait Time:                  7283s     121.39m     2.02h    0.08d  0.000 y
# Average job time:                 168s       2.79m     0.05h    0.00d
# Longest job:                      642s      10.70m     0.18h    0.01d
# Submission to last job:           642s      10.70m     0.18h    0.01d

    # translate sorted axt files into psl
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.rn3
    mkdir pslChrom
    set tbl = "blastzRn3"
    foreach f (axtChrom/chr*.axt)
      set c=$f:t:r
      echo "Processing chr $c"
      /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
    end
    #	That takes about 30 minutes

    # Load database tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.rn3/pslChrom
    for I in *.psl
    do
	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
	echo "done: ${I}"
    done
    # this is a 55 minute job
    #	Check results
    #	featureBits hg16 blastzRn3
    #	1013603401 bases of 2865248791 (35.376%) in intersection
    #	featureBits hg17 blastzRn3
    #	1013003285 bases of 2866216770 (35.343%) in intersection

# CHAIN RN3 BLASTZ (DONE - 2004-06-14 - Hiram)
#  re-worked with no 'axtFilter -notQ_random' on the axtChain step - 2004-06-23
#    used to be: axtFilter -notQ_random $1 | axtChain stdin

# The axtChain is best run on the small kluster, or the kk9 kluster
    ssh kki
    mkdir -p /cluster/data/hg17/bed/blastz.rn3/axtChain/run1
    cd /cluster/data/hg17/bed/blastz.rn3/axtChain/run1
    mkdir out chain

    ls -1S /cluster/data/hg17/bed/blastz.rn3/axtChrom/*.axt > input.lst
    cat << '_EOF_' > gsub
#LOOP
doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} out/$(root1).out
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    cat << '_EOF_' > doChain
#!/bin/csh
    axtChain $1 \
	/iscratch/i/gs.18/build35/bothMaskedNibs \
	/iscratch/i/rn3/bothMaskedNibs $2 > $3
'_EOF_'
    # << this line makes emacs coloring happy
    chmod a+x doChain

    # 46 jobs
    gensub2 input.lst single gsub jobList
    para create jobList
    para try
    para push # ... etc ...
# Completed: 46 of 46 jobs
# CPU time in finished jobs:       4645s      77.41m     1.29h    0.05d  0.000 y
# IO & Wait Time:                  6840s     114.00m     1.90h    0.08d  0.000 y
# Average job time:                 250s       4.16m     0.07h    0.00d
# Longest job:                     1539s      25.65m     0.43h    0.02d
# Submission to last job:          3761s      62.68m     1.04h    0.04d


    # now on the file server, sort chains
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.rn3/axtChain
    time chainMergeSort run1/chain/*.chain > all.chain
    #	real    36m42.170s
    #	user    4m55.970s
    #	sys     1m49.840s

    time chainSplit chain all.chain
    #	real    13m54.860s
    #	user    4m50.370s
    #	sys     1m3.260s

    # optionally: rm run1/chain/*.chain

    # Load chains into database
    # next machine
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.rn3/axtChain/chain
    foreach i (*.chain)
        set c = $i:r
        hgLoadChain hg17 ${c}_chainRn3 $i
        echo done $c
    end
    #	featureBits hg17 chainRn3
    #	2827052992 bases of 2866216770 (98.634%) in intersection
    #	(with filter:) 2826192649 bases of 2866216770 (98.604%) in intersection
    #	featureBits hg16 chainRn3
    #	2830563493 bases of 2865248791 (98.789%) in intersection

# NET RN3 (DONE - 2004-06-15 - Hiram)
#	Re-done due to Chain being re-done 2004-06-23
#       NOTE: Redo net axt's and net maf's to fix overlaps,
#       (using 8/05 netToAxt). (2005-08-16 kate)

    ssh eieio
    cd /cluster/data/hg17/bed/blastz.rn3/axtChain
    mkdir preNet
    cd chain
    foreach i (*.chain)
      echo preNetting $i
      /cluster/bin/i386/chainPreNet $i /cluster/data/hg17/chrom.sizes \
                        /cluster/data/rn3/chrom.sizes ../preNet/$i
    end

    cd ..
    mkdir n1
    cd preNet
    foreach i (*.chain)
      set n = $i:r.net
      echo primary netting $i
      /cluster/bin/i386/chainNet $i -minSpace=1 /cluster/data/hg17/chrom.sizes \
                            /cluster/data/rn3/chrom.sizes ../n1/$n /dev/null
    end

    cd ..
    cat n1/*.net | /cluster/bin/i386/netSyntenic stdin hNoClass.net
    #	memory usage 2510467072, utime 19307 s/100, stime 3181

    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.rn3/axtChain
    time netClass hNoClass.net hg17 rn3 rat.net \
	-tNewR=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInRat \
	-qNewR=/cluster/bluearc/rat/rn3/linSpecRep.notInHuman
    #	real    34m29.829s
    #	user    11m30.440s
    #	sys     1m52.730s

    # If things look good do
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.rn3/axtChain
    rm -r n1 hNoClass.net
    # Make a 'syntenic' subset of these with
    time netFilter -syn rat.net > ratSyn.net
    #	real    16m25.640s
    #	user    7m41.330s
    #	sys     1m1.150s

    # Load the nets into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.rn3/axtChain
    netFilter -minGap=10 rat.net |  hgLoadNet hg17 netRn3 stdin
    netFilter -minGap=10 ratSyn.net | hgLoadNet hg17 syntenyNetRn3 stdin
    #	real    37m0.199s
    #	user    15m13.770s
    #	sys     1m41.540s

    # check results
    # featureBits hg17 netRn3
    # 2817656275 bases of 2866216770 (98.306%) in intersection
    # (with axtFilter) 2816623107 bases of 2866216770 (98.270%) in intersection
    # featureBits hg16 netRn3
    # 2820958389 bases of 2865248791 (98.454%) in intersection

    # featureBits hg17 syntenyNetRn3
    # 2781748096 bases of 2866216770 (97.053%) in intersection
    # (with axtFilter) 2780883450 bases of 2866216770 (97.023%) in intersection
    # featureBits hg16 syntenyNetRn3
    # 2784011730 bases of 2865248791 (97.165%) in intersection

    # Add entries for net and chain to rat/hg17 trackDb

    # make net
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.rn3/axtChain
    mkdir ratNet
    time netSplit rat.net ratNet
    #	real    12m1.478s
    #	user    8m35.050s
    #	sys     1m7.230s

    # extract axts from net 
    mkdir ../axtNet ../mafNet
cat << 'EOF' > makeMaf.csh
    foreach n (ratNet/chr*.net)
	set c=$n:t:r
        echo $c
	netToAxt ratNet/$c.net chain/$c.chain \
		/cluster/data/hg17/nib  /cluster/data/rn3/nib stdout | \
                    axtSort stdin ../axtNet/$c.axt
        axtToMaf ../axtNet/$c.axt \
            /cluster/data/hg17/chrom.sizes /cluster/data/rn3/chrom.sizes \
                ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=rn3.
    end
'EOF'
    csh makeMaf.csh >&! makeMaf.log &
    tail -100f makeMaf.log
    mkdir -p /cluster/bluearc/hg17/mafNet
    cp -rp ../mafNet /cluster/bluearc/hg17/mafNet/rn3

    ssh hgwdev
    mkdir -p /cluster/data/hg17/bed/blastz.rn3/axtBest
    cd /cluster/data/hg17/bed/blastz.rn3/axtBest
    ln -s ../axtNet/chr*.axt .

    # copy net axt's to download area
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.rn3/axtNet
    mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/axtNet
    cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/axtNet
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/axtNet
    gzip *.axt
    # add README.txt file to dir (use previous assembly's copy as template)

    #  Convert those axt files to psl
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.rn3
    mkdir pslBest
    foreach a (axtBest/chr*.axt)
	set c=$a:t:r
	echo "processing $c.axt -> ${c}_blastzBestRn3.psl"
    /cluster/bin/i386/axtToPsl axtBest/${c}.axt \
	S1.len S2.len pslBest/${c}_blastzBestRn3.psl
	echo "Done: ${c}_blastzBestRn3.psl"
    end

    # Load tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.rn3/pslBest
    for I in chr*BestRn3.psl
    do
	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
	echo "done ${I}"
    done

     # check results
    # featureBits hg17 blastzBestRn3
    #	975533772 bases of 2866216770 (34.036%) in intersection
    # (with axtFilter) 970005525 bases of 2866216770 (33.843%) in intersection
    # featureBits hg16 blastzBestRn3
    #	976121391 bases of 2865248791 (34.068%) in intersection

    # Make /gbdb links and add them to the axtInfo table:
     mkdir -p /gbdb/hg17/axtBest/Rn3
     cd /gbdb/hg17/axtBest/Rn3
     ln -s /cluster/data/hg17/bed/blastz.rn3/axtNet/chr*.axt .
     cd /cluster/data/hg17/bed/blastz.rn3/axtNet
     rm -f axtInfoInserts.sql
     foreach f (/gbdb/hg17/axtBest/Rn3/chr*.axt)
       set chr=$f:t:r
       echo "INSERT INTO axtInfo (species, alignment, chrom, fileName) \
                VALUES ('rn3','Blastz Best in Genome','$chr','$f');" \
         >> axtInfoInserts.sql
     end
    hgsql hg17 < ~/kent/src/hg/lib/axtInfo.sql
    #	table axtInfo may already exist, ignore create error.
    hgsql hg17 < axtInfoInserts.sql


# MAKING RAT SYNTENY (DONE - 2004-06-30 - Hiram)
#	Re-Done after above done without the axtFilter

ssh hgwdev
mkdir /cluster/data/hg17/bed/syntenyRn3
cd /cluster/data/hg17/bed/syntenyRn3

# Copy all the needed scripts from /cluster/data/hg16/bed/syntenyMm3
cp -p /cluster/data/hg16/bed/syntenyMm3/*.pl .
cp -p /cluster/data/hg16/bed/syntenyMm3/*.sh .

./syntenicBest.pl -db=hg17 -table=blastzBestRn3
./smooth.pl
./joinsmallgaps.pl
./fillgap.pl -db=hg17 -table=blastzBestRn3
./synteny2bed.pl
#	The five commands above
#	real    196m2.565s
#	user    0m21.170s
#	sys     0m4.690s

#	Used to load this in syntenyRn3, but that type is misleading to
#	the table browser and fails the checkTableCoords check.
#	Better to use this ensRatMusHom type:
sed -e 's/ensPhusionBlast/ensRn3MusHom/g' \
      $HOME/kent/src/hg/lib/ensPhusionBlast.sql \
      > ensRn3MusHom.sql
hgLoadBed hg17 ensRn3MusHom ucsc100k.bed -sqlTable=ensRn3MusHom.sql

    #	featureBits hg17 ensRn3MusHom
    #	2592164486 bases of 2866216770 (90.439%) in intersection
    #	featureBits hg16 syntenyRn3
    #	2595919851 bases of 2865248791 (90.600%) in intersection


# MAKING RAT AXTTIGHT FROM AXTBEST (DONE - 2004-06-15 - Hiram)
    # After creating axtBest alignments above, use subsetAxt to get axtTight:
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.rn3/axtNet
    mkdir -p ../axtTight
    foreach i (*.axt)
      echo $i
      subsetAxt  $i ../axtTight/$i \
        ~kent/src/hg/mouseStuff/subsetAxt/coding.mat 3400
    end

    # translate to psl
    cd ../axtTight
    mkdir ../pslTight
    foreach i (*.axt)
      set c = $i:r
      axtToPsl $i ../S1.len ../S2.len ../pslTight/${c}_blastzTightRn3.psl
      echo "Done: $i"
    end

    # Load tables into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.rn3/pslTight
    for I in chr*TightRn3.psl
    do
	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
	echo "done ${I}"
    done

    #	Compare results with previous assembly
    #	featureBits hg17 blastzTightRn3
    #	153936720 bases of 2866216770 (5.371%) in intersection
    #	featureBits hg16 blastzTightRn3
    #	153151903 bases of 2865248791 (5.345%) in intersection

    # copy  axt's to download area
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.rn3/axtTight
    mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/axtTight
    cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/axtTight
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/axtTight
    gzip *.axt
    # add README.txt file to dir (use previous assembly's copy as template)

    # REDO downloads with fixed axtNet's (2005=09-13 kate)
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsRn3
    mv axtNet axtNet.old
    nice cp -rp /cluster/data/hg17/bed/blastz.rn3/axtNet .
    cd axtNet
    nice gzip *.axt
    md5sum *.axt.gz > md5sum.txt


# BLASTZ RN3 CLEAN UP (DONE - 2004-07-02 - Hiram)
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.rn3
    nice rm -rf raw &
    nice rm axtChain/run1/chain/* &
    nice rm -fr axtChain/n1 axtChain/hNoClass.net &
    nice gzip axtChrom/* pslChrom/* lav/*/* axtChain/all.chain axtChain/*.net &

# BLASTZ CHICKEN (GALGAL2) (DONE - 2004-06-14 - Fan)

    ssh kk
    mkdir /cluster/data/hg17/bed/blastz.galGal2.2004-06-14
    cd /cluster/data/hg17/bed
    ln -s /cluster/data/hg17/bed/blastz.galGal2.2004-06-14 blastz.galGal2
    cd blastz.galGal2
    # Set L=10000 (higher threshold on blastz's outer loop) and abridge 
    # repeats.
    cat << '_EOF_' > DEF
# human vs. chicken
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin

ALIGN=blastz-run
BLASTZ=blastz

# Specific settings for chicken (per Webb email to Brian Raney)
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=10000
BLASTZ_K=2200
BLASTZ_Q=/cluster/data/blastz/HoxD55.q
BLASTZ_ABRIDGE_REPEATS=1

# TARGET: Human
SEQ1_DIR=/iscratch/i/hg17/bothMaskedNibs
SEQ1_RMSK=
SEQ1_FLAG=
SEQ1_SMSK=/iscratch/i/gs.18/build35/linSpecRep.chicken
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Chicken
SEQ2_DIR=/iscratch/i/galGal2/nib
SEQ2_RMSK=
SEQ2_FLAG=
SEQ2_SMSK=/iscratch/i/galGal2/linSpecRep
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/cluster/store5/gs.18/build35/bed/blastz.galGal2

DEF=$BASE/DEF
RAW=$BASE/raw
CDBDIR=$BASE
SEQ1_LEN=$BASE/S1.len
SEQ2_LEN=$BASE/S2.len
'_EOF_'
    # << this line keeps emacs coloring happy

    # prepare first cluster run
    ssh kk
    cd /cluster/data/hg17/bed/blastz.galGal2
    bash
    # script copied over from /cluster/data/mm4/jkStuff/BlastZ_run0.sh
    #	it is a generic script and works for any assembly
    /cluster/data/hg17/jkStuff/BlastZ_run0.sh
    cd run.0
    para try, check, push, check, ....
Completed: 41943 of 41943 jobs
CPU time in finished jobs:   15330421s  255507.02m  4258.45h  177.44d  0.486 y
IO & Wait Time:                673809s   11230.15m   187.17h    7.80d  0.021 y
Average job time:                 382s       6.36m     0.11h    0.00d
Longest job:                     4651s      77.52m     1.29h    0.05d
Submission to last job:        169197s    2819.95m    47.00h    1.96d

    #	Second cluster run to convert the .out's to .lav's
    #	You do NOT want to run this on the big cluster.  It brings
    #	the file server to its knees.  Run this on the small cluster.
    ssh kki
    cd /cluster/data/hg17/bed/blastz.galGal2
    bash
    # script copied over from /cluster/data/mm4/jkStuff/BlastZ_run1.sh
    #	fixup machine check, should be kki, not kk
    /cluster/data/hg17/jkStuff/BlastZ_run1.sh
    cd run.1
    para try, check, push, etc ...
# Completed: 341 of 341 jobs
# CPU time in finished jobs:       1894s      31.56m     0.53h    0.02d  0.000 y
# IO & Wait Time:                  6271s     104.52m     1.74h    0.07d  0.000 y
# Average job time:                  24s       0.40m     0.01h    0.00d
# Longest job:                      131s       2.18m     0.04h    0.00d
# Submission to last job:           590s       9.83m     0.16h    0.01d

    #	Third cluster run to convert lav's to axt's
    cd /cluster/data/hg17/bed/blastz.galGal2
    #	The copy of this in mm4 was broken, fixed here
    /cluster/data/hg17/jkStuff/BlastZ_run2.sh
    cd run.2
    para try, check, push, etc ...
# Completed: 46 of 46 jobs
# CPU time in finished jobs:        426s       7.09m     0.12h    0.00d  0.000 y
# IO & Wait Time:                  7283s     121.39m     2.02h    0.08d  0.000 y
# Average job time:                 168s       2.79m     0.05h    0.00d
# Longest job:                      642s      10.70m     0.18h    0.01d
# Submission to last job:           642s      10.70m     0.18h    0.01d

    # translate sorted axt files into psl
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.galGal2
    mkdir pslChrom
    set tbl = "blastzGalGal2"
    foreach f (axtChrom/chr*.axt)
      set c=$f:t:r
      echo "Processing chr $c"
      /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
    end
    #	That takes about 30 minutes

    # Load database tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.galGal2/pslChrom
    bash
    for I in *.psl
    do
        /cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
        echo "done: ${I}"
    done   


# GNF ATLAS 2 (DONE - 2004-07-14 - Hiram
    # Align probes from GNF1H chip.
    ssh kk
    cd /cluster/data/hg17/bed
    mkdir -p geneAtlas2/run/psl
    cd geneAtlas2/run
    #	This bluearc/geneAtlas2 directory already exists
    # mkdir -p /cluster/bluearc/geneAtlas2
    # cp /projects/compbio/data/microarray/geneAtlas2/human/gnf1h.fa /cluster/bluearc/geneAtlas2
    ls -1 /scratch/hg/gs.18/build35/maskedContigs > genome.lst
    ls -1 /cluster/bluearc/geneAtlas2/gnf1h.fa > mrna.lst
    cat << '_EOF_' > gsub
#LOOP
blat -fine -ooc=/scratch/hg/h/11.ooc  /scratch/hg/gs.18/build35/maskedContigs/$(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    gensub2 genome.lst mrna.lst gsub jobList
    para create jobList
    para try
    para check
    para push
    para time
# Completed: 380 of 380 jobs
# CPU time in finished jobs:      10599s     176.65m     2.94h    0.12d  0.000 y
# IO & Wait Time:                  3893s      64.88m     1.08h    0.05d  0.000 y
# Average job time:                  38s       0.64m     0.01h    0.00d
# Longest job:                      649s      10.82m     0.18h    0.01d
# Submission to last job:           663s      11.05m     0.18h    0.01d

    # Do sort, best in genome filter, and convert to chromosome coordinates
    # to create gnf1h.psl.
    pslSort dirs raw.psl tmp psl
    pslReps -minCover=0.3 -minAli=0.95 -nearTop=0.005 raw.psl \
	contig.psl /dev/null
    #	Processed 80818 alignments
    liftUp ../affyGnf1h.psl ../../../jkStuff/liftAll.lft warn contig.psl
    rm -r contig.psl raw.psl psl

    # Load probes and alignments from GNF1H into database.
    ssh hgwdev
    cd /cluster/data/hg17/bed/geneAtlas2
    #	Already symlinked
    # ln -s /projects/compbio/data/microarray/geneAtlas2/human/gnf1h.fa \
    #	/gbdb/hgFixed/affyProbes
    hgLoadPsl hg17 affyGnf1h.psl
    hgLoadSeq hg17 /gbdb/hgFixed/affyProbes/gnf1h.fa

    grep -v U133B ../affyUclaNorm/hg17.affyU133AB_all.lifted.pslReps.psl \
	| sed -e "s/exemplar://; s/consensus://; s/U133A://" \
	| sed -e "s/;//" > affyU133A.psl

    hgMapMicroarray gnfAtlas2.bed hgFixed.gnfHumanAtlas2MedianRatio \
    	affyU133A.psl  /cluster/data/hg17/bed/geneAtlas2/affyGnf1h.psl

# Loaded 44696 rows of expression data from hgFixed.gnfHumanAtlas2MedianRatio
# Mapped 32857,  multiply-mapped 1462, missed 49, unmapped 11839

    hgLoadBed hg17 gnfAtlas2 gnfAtlas2.bed
    #	Loaded 34319 elements of size 15

# LOAD SNPS ( Daryl Thomas; November 7, 2004 ; 
	      snpExceptions added January 8, 2005 ; 
	      updated to build 124 on January 13, 2005;
	      added affy snps March 5, 2004 )
    set db    = hg17
    set org   = human
    set build = 124
    set dir   = /cluster/bluearc/snp/$db/build$build

    # ssh to some quiet machine with fast access to the bluearc
    # it takes ~4.5 hours to download the data 
    # (build 124 directly to /cluster/bluearc/... from eieio)
    # Check to make sure the chrMT file is included
    mkdir -p $dir $dir/ds_ch.xml $dir/det $dir/str $dir/loc $dir/seq
    cd $dir
    ln -s /cluster/data/$db/jkStuff/liftAll.lft .

    screen
    ftp ftp.ncbi.nih.gov
    cd snp/$org/XML
    prompt
    mget ds_ch*.xml.gz
    exit # screen
    exit # machine

    # TODO: check chromStart for each locType

    cp -f {$HOME}/kent/src/hg/snp/parseDbSnpXML /cluster/bin/scripts
    chmod 775 /cluster/bin/scripts/parseDbSnpXML

    #ssh kk
    touch jobList
    foreach file ( /cluster/bluearc/snp/$db/build$build/ds_ch*.xml.gz )
	    set out = $file:t:r
	    echo /cluster/bin/scripts/parseDbSnpXML $file /cluster/bluearc/snp/$db/build$build $out.contig >> jobList
    end

    # I removed ds_chMulti.xml.gz and ds_chNotOn.xml.gz from the job list
    # para create jobList; para push; para check ... 

#Completed: 25 of 25 jobs
#CPU time in finished jobs:      30120s     502.01m     8.37h    0.35d  0.001 y
#IO & Wait Time:                  2533s      42.21m     0.70h    0.03d  0.000 y
#Average job time:                1306s      21.77m     0.36h    0.02d
#Longest job:                     2611s      43.52m     0.73h    0.03d
#Submission to last job:          2611s      43.52m     0.73h    0.03d
    exit # kk

    mv -r $dir /cluster/data/$db/bed/snp/build$build
    set dir = /cluster/data/$db/bed/snp/build$build
    cd $dir

    # concatenate the details files to make it easier to lift (and load)
    time zcat det/ds_ch*.xml.contig.det.gz > $db.build$build.contig.bed
    # 33.380u 24.470s 1:54.79 50.3%   0+0k 0+0io 86pf+0w (hgwdev)
    time gzip $db.build$build.contig.bed
    # 251.160u 16.770s 12:40.77 35.2% 0+0k 0+0io 83pf+0w (hgwdev/bluearc - should have done it on eieio/store5)

    # some of the NT contigs are not in the liftSpec - this is expected as snps that map to
    # alternate assemblies (Celera) are in the original files, but we disregard their mappings.
    time liftUp $db.build$build.bed liftAll.lft warn $db.build$build.contig.bed.gz 
    # 232.260u 30.050s 5:09.04 84.8%  0+0k 0+0io 379pf+0w (hgwdev/store5)

    time gzip hg17.build124.bed                                                                      
    # 141.980u 8.180s 2:34.43 97.2%   0+0k 0+0io 83pf+0w

    # hgLoadBed is the important step - check to make sure there are no warnings
    time hgLoadBed $db snp $db.build$build.bed.gz -sqlTable=${HOME}/kent/src/hg/lib/snp.sql
    # Loaded 9131054 elements of size 16
    # 225.040u 37.030s 35:20.45 12.3% 0+0k 0+0io 308pf+0w

    # basic snp table is now loaded, but exception column needs to be updated
    # ~ 3 hours wall clock time from here to end

    # run queries from snpException.query against snp table
    mkdir -p /usr/local/apache/htdocs/qa/test-results/snpException/build$build
    cd       /usr/local/apache/htdocs/qa/test-results/snpException/build$build
    time snpException hg17 0 ${db}snpException > ${db}snpException.log
    chmod o+rx . 
    chmod o+r  * 
    # 10.610u 19.200s 53:59.98 0.9%   0+0k 0+0io 264pf+0w

    # check alignment of flanking sequences
    time snpValid $db /cluster/data/$db/bed/snp/build$build/seq > ${db}snpValid.log
    # 5205.860u 216.570s 1:55:10.27 78.4%     0+0k 0+0io 72408pf+0w (hgwdev)

    ### NOTE: the pseudoautosomal snps are reported in the chrX files
    ### only, which causes problems for snpValid when checking the
    ### chrY snp mappings.  I got around this by confirming that all
    ### of the 'missing flank' errors (#23) were in pseudoautosomal
    ### regions and ignoring them.  I manually truncated the
    ### hg17snpException.23.bed file before continuing with the next
    ### step.  This could/should be fixed in the next iteration.

    # create list of statements to update the snp table and run them
    time tail +3 ${db}snpException.* | awk '/rs/ {printf "%s\t%d\t%d\n",$4,$2,$5}' | sort -k1,2n > exceptionList.txt
    # ~10 seconds
    time updateExceptionList.pl < exceptionList.txt > updateExceptionList.sql
    # 7.250u 0.390s 0:07.87 97.0%     0+0k 0+0io 337pf+0w
    time hgsql hg17 < updateExceptionList.sql
    # 8.420u 10.370s 11:58.44 2.6%    0+0k 0+0io 413pf+0w build123 (this is mostly a mysql process)
    # 6.550u  9.370s 14:34.17 1.8%    0+0k 0+0io 413pf+0w build124
    # > wc -l build12*/updateExceptionList.sql
    #  387166 build123/updateExceptionList.sql
    #  383759 build124/updateExceptionList.sql

    # Add Affy SNPs from new submission 
    #!/bin/csh -fe
    # rm -f log ; date ; ./loadAffySnps.csh > & log ; date ; cat log

    set db = hg17
    cd /cluster/data/$db/bed/snp/affy/latest
    touch affy.txt affy.bed Affy.bed bed.tab
    rm -f affy*.txt affy*.bed Affy.bed* bed.tab

    # datafile was provided by Valmeekam, Venu [Venu_Valmeekam@affymetrix.com]
    tar xfz affyhg17maps_withstrand_alleles.tgz
    wc -l affy*txt

    awk '$1 !~ /^chrom/ {printf("chr%s\t%d\t%d\t%s\t0\t%s\t%s\tunknown\tsnp\tunknown\t0\t0\tunknown\texact\tAffy10K\t0\n",        $1,$2,$3,$4,$6,$7);}' < affy10K.txt         > affy10K.bed
    awk '$1 !~ /^chrom/ {printf("chr%s\t%d\t%d\t%s\t0\t%s\t%s\tunknown\tsnp\tunknown\t0\t0\tunknown\texact\tAffy10Kv2\t0\n",      $1,$2,$3,$4,$6,$7);}' < affy10Kv2.txt       > affy10Kv2.bed
    awk '$1 !~ /^chrom/ {printf("chr%s\t%d\t%d\t%s\t0\t%s\t%s\tunknown\tsnp\tunknown\t0\t0\tunknown\texact\tAffy50K_HindIII\t0\n",$1,$2,$3,$4,$6,$7);}' < affy50K_HindIII.txt > affy50K_HindIII.bed
    awk '$1 !~ /^chrom/ {printf("chr%s\t%d\t%d\t%s\t0\t%s\t%s\tunknown\tsnp\tunknown\t0\t0\tunknown\texact\tAffy50K_XbaI\t0\n",   $1,$2,$3,$4,$6,$7);}' < affy50K_XbaI.txt    > affy50K_XbaI.bed
    
    # this is a temporary kluge to fix some bad input data.
    cat affy*.bed | sed 's/_par//' > Affy.bed
    
    # the source enum for 'dbSnp' is 2; all of the affy* values are higher.
    hgsql $db -e "delete from snp where source > 2 "
    
    hgLoadBed $db snp Affy.bed -oldTable -tab
    
    rm -f affy*.txt affy*.bed bed.tab
    gzip Affy.bed
    
    #mysql> select source, count(*) from hg17.snp group by source;                                                                                                                   
    #+-----------------+----------+
    #| source          | count(*) |
    #+-----------------+----------+
    #| dbSnp           |  9131054 |
    #| Affy10K         |    11344 |
    #| Affy10Kv2       |    10032 |
    #| Affy50K_HindIII |    56859 |
    #| Affy50K_XbaI    |    58494 |
    #+-----------------+----------+

    #March 7, 2005: fix pseudoautosomal snps:
    #SNP_A-1606360
    #SNP_A-1606329
    #SNP_A-1666553
    #SNP_A-1715750
    #SNP_A-1726331
    #SNP_A-1685712
    #SNP_A-1735899
    #SNP_A-1726272
    #SNP_A-1660936
    #SNP_A-1662285
    #SNP_A-1680848
    #SNP_A-1671440
    #SNP_A-1719355
    #SNP_A-1716499
    #SNP_A-1643847
    #SNP_A-1646007
    #SNP_A-1715285
    #SNP_A-1657714
    #SNP_A-1725038
    #SNP_A-1713938
    #SNP_A-1708565
    #SNP_A-1510243
    #SNP_A-1510197
    #SNP_A-1606356

    delete from snp 
    where  chrom = 'chrY' 
    and    name in ('SNP_A-1606360','SNP_A-1606329','SNP_A-1666553','SNP_A-1715750','SNP_A-1726331','SNP_A-1685712','SNP_A-1735899','SNP_A-1726272','SNP_A-1660936','SNP_A-1662285','SNP_A-1680848','SNP_A-1671440','SNP_A-1719355','SNP_A-1716499','SNP_A-1643847','SNP_A-1646007','SNP_A-1715285','SNP_A-1657714','SNP_A-1725038','SNP_A-1713938','SNP_A-1708565','SNP_A-1510243','SNP_A-1510197','SNP_A-1606356');

    update snp 
    set chrom = 'chrX' 
    where name in ('SNP_A-1606360','SNP_A-1606329','SNP_A-1666553','SNP_A-1715750','SNP_A-1726331','SNP_A-1685712','SNP_A-1735899','SNP_A-1726272','SNP_A-1660936','SNP_A-1662285','SNP_A-1680848','SNP_A-1671440','SNP_A-1719355','SNP_A-1716499','SNP_A-1643847','SNP_A-1646007','SNP_A-1715285','SNP_A-1657714','SNP_A-1725038','SNP_A-1713938','SNP_A-1708565','SNP_A-1510243','SNP_A-1510197','SNP_A-1606356');

    insert into snp 
	select  bin, 'chrY' as chrom, chromStart, chromEnd, name, score, strand, 
		observed, molType, class, valid, avHet, avHetSE, func, locType, source, exception 
	from    snp 
	where   name in ('SNP_A-1606360','SNP_A-1606329','SNP_A-1666553','SNP_A-1715750','SNP_A-1726331','SNP_A-1685712','SNP_A-1735899','SNP_A-1726272','SNP_A-1660936','SNP_A-1662285','SNP_A-1680848','SNP_A-1671440','SNP_A-1719355','SNP_A-1716499','SNP_A-1643847','SNP_A-1646007','SNP_A-1715285','SNP_A-1657714','SNP_A-1725038','SNP_A-1713938','SNP_A-1708565','SNP_A-1510243','SNP_A-1510197','SNP_A-1606356');

    select chrom, count(*) 
    from snp 
    where name in ('SNP_A-1606360','SNP_A-1606329','SNP_A-1666553','SNP_A-1715750','SNP_A-1726331','SNP_A-1685712','SNP_A-1735899','SNP_A-1726272','SNP_A-1660936','SNP_A-1662285','SNP_A-1680848','SNP_A-1671440','SNP_A-1719355','SNP_A-1716499','SNP_A-1643847','SNP_A-1646007','SNP_A-1715285','SNP_A-1657714','SNP_A-1725038','SNP_A-1713938','SNP_A-1708565','SNP_A-1510243','SNP_A-1510197','SNP_A-1606356') 
    group by chrom;;


## LS-SNP links [load data only] (Daryl Thomas; November 3, 2005)
    # Data from Rachel Karchin in the Andrej Sali lab at UCSF
    # /cluster/data/hg17/bed/lssnp
    hgsql hg17 < ${HOME}/kent/src/hg/lib/lsSnpFunction.sql
    hgsql hg17 < ${HOME}/kent/src/hg/lib/lsSnpStructure.sql
    mysql> load data local infile "snp-human3-function-predictions.txt" into table lsSnpFunction;
    Query OK, 24337 rows affected (1.27 sec)
    mysql> load data local infile "snp-human3-structure-predictions.txt" into table lsSnpStructure;
    Query OK, 34764 rows affected (2.36 sec)

# Tajima's D (DONE -- 2005-09-20 -- Daryl)
# Data from Chris Carlson in Debbie Nickerson's lab
# Chris Carlson [csc47<AT>u<DOT>washington<DOT>edu]

    # get data from ftp site, unpack in $dir:
    # tar tvfz *gz | more
    # -rw-r--r-- chris/admin 34405061 2005-06-03 13:22:15 AD.SNP.track
    # -rw-r--r-- chris/admin 29869512 2005-06-03 13:22:30 ED.SNP.track
    # -rw-r--r-- chris/admin 27154049 2005-06-03 13:22:41 XD.SNP.track
    # -rw-r--r-- chris/admin 10948753 2005-06-02 21:12:27 AD.tajd.track
    # -rw-r--r-- chris/admin 10928630 2005-06-02 21:12:39 ED.tajd.track
    # -rw-r--r-- chris/admin 10926122 2005-06-02 21:12:51 XD.tajd.track

    set db=hg17
    set dir=/cluster/data/$db/bed/tajdpoly/latest
    cd $dir
    
    tar xvfz TajDtracks.tar.gz

    mac2unix < AD.SNP.track  | grep -v track | sed 's/1\.02e+08/102000000/;s/8\.8e+07/88000000/;s/1\.5e+07/15000000/' > hg17.tajdSnpAd.bed
    mac2unix < ED.SNP.track  | grep -v track | sed 's/1\.02e+08/102000000/;s/8\.8e+07/88000000/;s/1\.5e+07/15000000/' > hg17.tajdSnpEd.bed
    mac2unix < XD.SNP.track  | grep -v track | sed 's/1\.02e+08/102000000/;s/8\.8e+07/88000000/;s/1\.5e+07/15000000/' > hg17.tajdSnpXd.bed
    mac2unix < AD.tajd.track | grep -v track | awk '{printf"%s\t%s\t%d\t%.3f\n",$1,$2,$3,$4}' > hg17.tajdAd.bedGraph
    mac2unix < ED.tajd.track | grep -v track | awk '{printf"%s\t%s\t%d\t%.3f\n",$1,$2,$3,$4}' > hg17.tajdEd.bedGraph
    mac2unix < XD.tajd.track | grep -v track | awk '{printf"%s\t%s\t%d\t%.3f\n",$1,$2,$3,$4}' > hg17.tajdXd.bedGraph
    
    set chain = /cluster/data/hg17/bed/bedOver/hg17ToHg16.over.chain

    foreach pop (Ad Ed Xd)
	liftOver hg17.tajdSnp$pop.bed   $chain hg16.tajdSnp$pop.bed   hg17ToHg16.tajdSnp$pop.unmapped
	liftOver hg17.tajd$pop.bedGraph $chain hg16.tajd$pop.bedGraph hg17ToHg16.tajd$pop.unmapped
	foreach db (hg16 hg17)
	    hgLoadBed -bedGraph=4 $db tajd$pop    $db.tajd$pop.bedGraph
	    hgLoadBed             $db tajdSnp$pop $db.tajdSnp$pop.bed
	end
    end

    set where1 = "where t.bin=g.bin and t.chrom=g.chrom and (t.chromStart between g.chromStart and g.chromEnd or t.chromEnd between g.chromStart and g.chromEnd)"
    set where2 = "t, chromInfo c where t.chromStart < 0 or (t.chrom=c.chrom and t.chromEnd > c.size)"
    set list = "as pop, t.chrom, t.chromStart from"
    foreach db (hg16 hg17)
	rm -f $db.delete.sql
	touch $db.delete.sql
	foreach p (Ad Ed Xd SnpAd SnpEd SnpXd)
	    foreach c (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y)
		echo "select 'tajd$p' $list tajd${p} t,chr${c}_gap g $where1" | \
		    hgsql $db | \
		    grep -v pop | \
		    awk '{printf "delete from %s where chrom=\"%s\" and chromStart=%d;\n",$1,$2,$3}' \
		    >> $db.delete.sql
	    end
	    echo "select 'tajd$p' $list tajd${p} $where2" | \
		hgsql $db | \
		grep -v pop | \
		awk '{printf "delete from %s where chrom=\"%s\" and chromStart=%d;\n",$1,$2,$3}'\
		>> $db.delete.sql
	end
	hgsql $db < $db.delete.sql
    end


# GENE SORTER (AKA: FAMILY BROWSER) (DONE - 2004-06-16 - Hiram)
# Added knownToU133Plus2 track (2004-10-14)
#	to be done after knownGene tables are complete from known gene
#	process.
#
# Cluster together various alt-splicing isoforms.
#	Creates the knownIsoforms and knownCanonical tables
ssh hgwdev
mkdir /cluster/data/hg17/bed/geneSorter.2004-06-15
ln -s /cluster/data/hg17/bed/geneSorter.2004-06-15 \
	/cluster/data/hg17/bed/geneSorter
cd /cluster/data/hg17/bed/geneSorter
hgClusterGenes hg17 knownGene knownIsoforms knownCanonical

# Extract peptides from knownGenes into fasta file
# and create a blast database out of them.
mkdir /cluster/data/hg17/bed/geneSorter/blastp
cd /cluster/data/hg17/bed/geneSorter/blastp
pepPredToFa hg17 knownGenePep known.faa
#	You may need to build this binary in src/hg/near/pepPredToFa
/scratch/blast/formatdb -i known.faa -t known -n known
#	This command is in /projects/compbio/bin/$MACH/formatdb

# Copy over database to bluearc
rm -fr /cluster/bluearc/hg17/blastp
mkdir -p /cluster/bluearc/hg17/blastp
cp -p /cluster/data/hg17/bed/geneSorter/blastp/known.* \
	/cluster/bluearc/hg17/blastp

#	Had to pick up a new blastall binary (2004-06-15)
#	Our old one would no longer run on our systems that have
#	updated Linux versions
mkdir /cluster/bluearc/blast229
cd /cluster/bluearc/blast229
wget --timestamping \
    ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/2.2.9/blast-2.2.9-ia32-linux.tar.gz
wget --timestamping \
    ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/2.2.9/ChangeLog.txt
wget --timestamping \
    ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/2.2.9/ReleaseNotes.txt
tar xvzf blast-2.2.9-ia32-linux.tar.gz


# Split up fasta file into bite sized chunks for cluster
cd /cluster/data/hg17/bed/geneSorter/blastp
mkdir split
faSplit sequence known.faa 8000 split/kg

# Make parasol run directory
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/self
cd /cluster/data/hg17/bed/geneSorter/blastp/self
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
	-p blastp -d /cluster/bluearc/hg17/blastp/known -i $1 -o $2 \
	-e 0.01 -m 8 -b 1000
'_EOF_'
    # << keep emacs happy
chmod +x blastSome

# Make gensub2 file
cat  << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#	'ls ../../split/*.fa' is too much, hence the echo
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
# Wait a couple of minutes, and do a para check,  if all is good
# then do a
para push
# This should finish in ~15 minutes if the cluster is free.
Completed: 7749 of 7749 jobs
CPU time in finished jobs:     182148s    3035.81m    50.60h    2.11d  0.006 y
IO & Wait Time:                 22954s     382.56m     6.38h    0.27d  0.001 y
Average job time:                  26s       0.44m     0.01h    0.00d
Longest job:                      372s       6.20m     0.10h    0.00d
Submission to last job:           871s      14.52m     0.24h    0.01d

# Load into database.  This takes about 30 minutes
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/self/run/out
time hgLoadBlastTab hg17 knownBlastTab *.tab
# Scanning through 7749 files
# Loading database with 11799667 rows
#	Hg16 was:       11376875 rows
# real    30m10.761s
# user    5m25.490s
# sys     1m0.630s

cd /cluster/data/hg17/bed/geneSorter
# Create table that maps between known genes and RefSeq
hgMapToGene hg17 refGene knownGene knownToRefSeq
#	may need to build this command in src/hg/near/hgMapToGene
#	hgsql -e "select count(*) from knownToRefSeq;" hg17
#	row count changed from 36078 in Hg16 to 36082

# Create table that maps between known genes and LocusLink
hgsql --skip-column-names -e "select mrnaAcc,locusLinkId from refLink" hg17 \
	> refToLl.txt
hgMapToGene hg17 refGene knownGene knownToLocusLink -lookup=refToLl.txt
#	hgsql -e "select count(*) from knownToLocusLink;" hg17
#	row count went from 36078 in Hg16 to 36082

# Create table that maps between known genes and Pfam domains
hgMapViaSwissProt hg17 knownGene name proteinID Pfam knownToPfam
#	hgsql -e "select count(*) from knownToPfam;" hg17
#	row count dropped from 30467 in Hg16 to 29725


# Create table to map between known genes and GNF Atlas2
# expression data.
    hgMapToGene hg17 gnfAtlas2 knownGene knownToGnfAtlas2 '-type=bed 12'
#	hgsql -e "select count(*) from knownToGnfAtlas2;" hg17
#	row count droppted from 35817 in Hg16 to 35739

# Create expression distance table - takes about an hour
    hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
    	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
	-lookup=knownToGnfAtlas2
# Got 35739 unique elements in hgFixed.gnfHumanAtlas2MedianRatio
    #	hgsql -e "select count(*) from gnfAtlas2Distance;" hg17
    #	row count went from 35,817,000 in Hg16 to 35,739,000
    #	real    108m1.671s
    #	user    89m30.680s
    #	sys     3m6.800s

# Create a table that maps between known genes and 
# the nice affy expression data.
hgMapToGene "-type=bed 12" hg17 affyUclaNorm knownGene knownToU133
    #	hgsql -e "select count(*) from knownToU133;" hg17
    #	row count went from 37,634 in Hg16 to 36,795

# Create expression distance table.  This will take about 2.5 hours
cd /tmp
cp -p ~/kent/src/hg/near/hgExpDistance/affyUcla.weight .
time hgExpDistance hg17 affyUclaNorm affyUclaExp knownExpDistance \
	-weights=affyUcla.weight -lookup=knownToU133
    #	211 genes, 42 weights, 26.500000 total wieght
    #	Got 36795 unique elements in affyUclaNorm
    #	real    154m1.058s
    #	user    134m45.000s
    #	sys     3m1.990s

# Create table that maps between known genes and 
# the GNF data.
cd /tmp
hgMapToGene hg17 affyU95 knownGene knownToU95
    #	row count went from 18780 in Hg16 to 18796
    #	hgFixed.gnfHumanU95Exps argument is unused, no need to exist
hgExpDistance hg17 hgFixed.gnfHumanU95MedianRatio \
	hgFixed.gnfHumanU95Exps gnfU95Distance  -lookup=knownToU95
    #	row count went from 17711000 in Hg16 to 17710000
    #	real    21m37.703s
    #	user    13m35.110s
    #	sys     0m28.470s

# Create known gene mapping table and expression distance tables
# for GNF Atlas 2.  (The hgExpDistance takes only 10 minutes.)

hgMapToGene hg17 affyGnf1h knownGene knownToGnf1h
hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
	-lookup=knownToGnf1h
# Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
# Got 9756 unique elements in hgFixed.gnfHumanAtlas2MedianRatio

# create table mapping knownGenes to affyU133Plus2 table (2004-10-14, hartera)
cd /cluster/data/hg17/bed/geneSorter
hgMapToGene hg17 affyU133Plus2 knownGene knownToU133Plus2

# Make sure that GO database is up to date.
See README in /cluster/store1/geneOntology.
#	I update this GO database very carefully, checking that all
#	structures in it remain the same from release to release and
#	backing up the current go DB in a backup database.  In this case
#	the backup is go040107 - when it was loaded for Mm4, and the new
#	go database is based on data from Dec 17th 2003 and Feb 2004 according
#	to the time stamp on the fetched data.  This build was done in
#	/cluster/store1/geneOntology/20040217

cd /cluster/data/hg17/bed/geneSorter

XXX - DO NOT YET HAVE ensGene table - must wait on Ensembl to release that
XXX - have not created the knownToEnsembl table yet - 2004-07-15 - Hiram
# Create knownToEnsembl column
hgMapToGene hg17 ensGene knownGene knownToEnsembl
#	table row count went from previous version: 36068 to 38251

# Make knownToCdsSnp table (DONE Nov 11, 2004, Heather)
  ssh hgwdev
  nice hgMapToGene hg17 snp knownGene knownToCdsSnp -all -cds
# row count 165728
# unique 34013
# approx. 5 minutes running time

# Make C. elegans ortholog column using blastp on wormpep.
# First make C. elegans protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This is already done, see makeMm3.doc for procedure
#	the directory: /cluster/bluearc/ce1/blastp should have data

#	The blast jobs below can be run on the kk or kk9 clusters
# Create the ceBlastTab
ssh kk9
mkdir /cluster/data/hg17/bed/geneSorter/blastp/ce1
cd /cluster/data/hg17/bed/geneSorter/blastp/ce1
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
	-p blastp -d /cluster/bluearc/ce1/blastp/wormPep \
	-i $1 -o $2 -e 0.01 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
#	Only takes 10 minutes on an idle cluster
# Completed: 7749 of 7749 jobs
# CPU time in finished jobs:      32023s     533.72m     8.90h    0.37d  0.001 y
# IO & Wait Time:                 20643s     344.05m     5.73h    0.24d  0.001 y
# Average job time:                   7s       0.11m     0.00h    0.00d
# Longest job:                      110s       1.83m     0.03h    0.00d
# Submission to last job:          1911s      31.85m     0.53h    0.02d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/ce1/run/out
hgLoadBlastTab hg17 ceBlastTab -maxPer=1 *.tab
#	row count went from 27620 to 27616

# Make mouse ortholog column using blastp on mouse known genes.
# First make mouse protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This already exists.  See makeMm5.doc for procedure
#	the directory: /cluster/bluearc/scratch/mus/mm5/blastp should have data

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/mm5
cd /cluster/data/hg17/bed/geneSorter/blastp/mm5
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
	-p blastp -d /cluster/bluearc/scratch/mus/mm5/blastp/known \
	-i $1 -o $2 -e 0.001 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#	this echo trick is used because otherwise the command line is
#	too long and you can not do a simple ls
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7749 of 7749 jobs
# CPU time in finished jobs:     139041s    2317.34m    38.62h    1.61d  0.004 y
# IO & Wait Time:                 21227s     353.79m     5.90h    0.25d  0.001 y
# Average job time:                  21s       0.34m     0.01h    0.00d
# Longest job:                      260s       4.33m     0.07h    0.00d
# Submission to last job:          1137s      18.95m     0.32h    0.01d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/mm5/run/out
hgLoadBlastTab hg17 mmBlastTab -maxPer=1 *.tab
# Scanning through 7748 files
#	row count went from 36471 to 36638

# Make rat ortholog column using blastp on rat known genes.
# First make rat protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This already exists.  See makeRn3.doc for procedure.
#	Files were put in this directory: /cluster/bluearc/rn3/blastp/

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/rn3
cd /cluster/data/hg17/bed/geneSorter/blastp/rn3
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
    -p blastp -d /cluster/bluearc/rn3/blastp/known \
    -i $1 -o $2 -e 0.001 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#	this echo trick is used because otherwise the command line is
#	too long and you can not do a simple ls
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
#Completed: 7749 of 7749 jobs
#CPU time in finished jobs:      31035s     517.25m     8.62h    0.36d  0.001 y
#IO & Wait Time:                 38472s     641.20m    10.69h    0.45d  0.001 y
#Average job time:                   9s       0.15m     0.00h    0.00d
#Longest job:                       75s       1.25m     0.02h    0.00d
#Submission to last job:           169s       2.82m     0.05h    0.00d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/rn3/run/out
hgLoadBlastTab hg17 rnBlastTab -maxPer=1 *.tab
# Scanning through 7749 files
#Loading database with 25574 rows

# Make Danio rerio (zebrafish) ortholog column using blastp on Ensembl.
# First make protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This is already done, see makeMm3.doc for procedure
#	the directory: /cluster/bluearc/dr1/blastp should have data

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/dr1
cd /cluster/data/hg17/bed/geneSorter/blastp/dr1
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
	-p blastp -d /cluster/bluearc/dr1/blastp/ensembl \
	-i $1 -o $2 -e 0.005 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7749 of 7749 jobs
# CPU time in finished jobs:     100217s    1670.28m    27.84h    1.16d  0.003 y
# IO & Wait Time:                 23697s     394.95m     6.58h    0.27d  0.001 y
# Average job time:                  16s       0.27m     0.00h    0.00d
# Longest job:                      233s       3.88m     0.06h    0.00d
# Submission to last job:          1667s      27.78m     0.46h    0.02d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/dr1/run/out
hgLoadBlastTab hg17 drBlastTab -maxPer=1 *.tab
#	row count went from 32971 to 33023

# Make Saccharomyces cerevisiae (yeast) ortholog column using blastp on RefSeq.
# First make protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This is already done, see makeMm3.doc for procedure
#	the directory: /cluster/bluearc/sc1/blastp should have data

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/sc1
cd /cluster/data/hg17/bed/geneSorter/blastp/sc1
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
	-p blastp -d /cluster/bluearc/sc1/blastp/sgd \
	-i $1 -o $2 -e 0.01 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7749 of 7749 jobs
# CPU time in finished jobs:      20738s     345.64m     5.76h    0.24d  0.001 y
# IO & Wait Time:                 22018s     366.96m     6.12h    0.25d  0.001 y
# Average job time:                   6s       0.09m     0.00h    0.00d
# Longest job:                       39s       0.65m     0.01h    0.00d
# Submission to last job:           572s       9.53m     0.16h    0.01d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/sc1/run/out
hgLoadBlastTab hg17 scBlastTab -maxPer=1 *.tab
#	row count went from 18286 to 18265

# Make Drosophila melanagaster ortholog column using blastp on FlyBase.
# First make SwissProt protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This is already done, see makeMm3.doc for procedure
#	the directory: /cluster/bluearc/dm1/blastp should have data

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/dm1
cd /cluster/data/hg17/bed/geneSorter/blastp/dm1
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
	-p blastp -d /cluster/bluearc/dm1/blastp/flyBase \
	-i $1 -o $2 -e 0.01 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7749 of 7749 jobs
# CPU time in finished jobs:      82022s    1367.03m    22.78h    0.95d  0.003 y
# IO & Wait Time:                 21982s     366.37m     6.11h    0.25d  0.001 y
# Average job time:                  13s       0.22m     0.00h    0.00d
# Longest job:                      174s       2.90m     0.05h    0.00d
# Submission to last job:          1439s      23.98m     0.40h    0.02d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/dm1/run/out
hgLoadBlastTab hg17 dmBlastTab -maxPer=1 *.tab
#	row count went from 29322 to 29341

####  Blat knownGene proteins to determine exons (braney 2004-06-20 DONE)
    ssh hgwdev
    cd /cluster/data/hg17/bed
    mkdir blat.hg17KG.2004-06-20
    rm blat.hg17KG
    ln -s  blat.hg17KG.2004-06-20 blat.hg17KG
    cd blat.hg17KG
    pepPredToFa hg17 knownGenePep known.fa
    hgPepPred hg17 generic blastKGPep00 known.fa
    grep ">" known.fa | sed "s/>//" > kgName.lst
    kgName hg17 kgName.lst blastKGRef00
    hgsql hg17 < ~/kent/src/lib/hg/blastRef.sql
    echo "rename table blastRef to blastKGRef00" | hgsql hg17
    echo "load data local infile 'blastKGRef00' into table blastKGRef00" | hgsql hg17
    ssh kk
    cd /cluster/data/hg17/bed/blat.hg17KG
    cat << '_EOF_' > blatSome
#!/bin/csh -fe
/cluster/bin/i386/blat -t=dnax -q=prot -out=pslx $1 $2 $3
'_EOF_'
    # << keep emacs happy
    chmod +x blatSome
    ls -1S /scratch/hg/gs.18/build35/bothMaskedNibs/*.nib > human.lst
    mkdir kgfa
    cd kgfa
    faSplit sequence ../known.fa 3000 kg
    cd ..
    ls -1S kgfa/*.fa > kg.lst
    cat << '_EOF_' > blatGsub
#LOOP
blatSome $(path1) {check in line $(path2)} {check out line psl/$(root1)/$(root2).psl}
#ENDLOOP
'_EOF_'
    # << keep emacs happy
    gensub2 human.lst kg.lst blatGsub blatSpec
    mkdir psl
    cd psl
    foreach i (`cat ../human.lst`)
	mkdir `basename $i .nib`
    end
    cd ..
    para create blatSpec
    para push

# Completed: 133676 of 133676 jobs
# CPU time in finished jobs:   29661130s  494352.16m  8239.20h  343.30d  0.941 y
# IO & Wait Time:               2181179s   36352.99m   605.88h   25.25d  0.069 y
# Average job time:                 238s       3.97m     0.07h    0.00d
# Longest job:                   105972s    1766.20m    29.44h    1.23d

    ssh eieio
    cd /cluster/data/hg17/bed/blat.hg17KG
    pslSort dirs raw.psl /tmp psl/*
    pslReps -nohead -minCover=0.9 -minAli=0.9 raw.psl cooked.psl /dev/null
    pslUniq cooked.psl hg17KG.psl
    pslxToFa hg17KG.psl hg17KG_ex.fa -liftTarget=genome.lft -liftQuery=protein.lft

# BLASTZ MM4 (DONE - 2004-06-22 - Hiram)
    ssh kk
    mkdir -p /cluster/data/hg17/bed/blastz.mm4.2004-06-21
    cd /cluster/data/hg17/bed
    ln -s  blastz.mm4.2004-06-21 blastz.mm4
    cd blastz.mm4

    cat << '_EOF_' > DEF
# human vs. mouse
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/home/angie/schwartzbin:/cluster/home/kent/bin/i386

ALIGN=blastz-run
BLASTZ=blastz
BLASTZ_H=2000
BLASTZ_ABRIDGE_REPEATS=1

# TARGET
# Human
SEQ1_DIR=/iscratch/i/gs.18/build35/bothMaskedNibs
# not used
SEQ1_RMSK=
# not used
SEQ1_FLAG=
SEQ1_SMSK=/iscratch/i/gs.18/build35/linSpecRep.notInRat
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY
# Mouse
SEQ2_DIR=/scratch/mus/mm4/softNib
# RMSK not currently used
SEQ2_RMSK=/scratch/mus/mm4/rmsk
# FLAG not currently used
SEQ2_FLAG=-rodent
SEQ2_SMSK=/scratch/mus/mm4/linSpecRep.notInHuman
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=30000000
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/blastz.mm4

DEF=$BASE/DEF
RAW=$BASE/raw
CDBDIR=$BASE
SEQ1_LEN=$BASE/S1.len
SEQ2_LEN=$BASE/S2.len
'_EOF_'
    # << this line keeps emacs coloring happy

    # prepare first cluster run
    ssh kk
    cd /cluster/data/hg17/bed/blastz.mm4
    /cluster/data/hg17/jkStuff/BlastZ_run0.sh
    cd run.0
    para try, check, push, check, ....
# Completed: 43648 of 43648 jobs
# CPU time in finished jobs:   16448001s  274133.36m  4568.89h  190.37d  0.522 y
# IO & Wait Time:                751666s   12527.76m   208.80h    8.70d  0.024 y
# Average job time:                 394s       6.57m     0.11h    0.00d
# Longest job:                     8323s     138.72m     2.31h    0.10d
# Submission to last job:         44244s     737.40m    12.29h    0.51d

    #	the file server to its knees.  Run this on the small cluster.
    ssh kki
    cd /cluster/data/hg17/bed/blastz.mm4
    /cluster/data/hg17/jkStuff/BlastZ_run1.sh
    cd run.1
    para try, check, push, etc ...
# Completed: 341 of 341 jobs
# CPU time in finished jobs:       3925s      65.42m     1.09h    0.05d  0.000 y
# IO & Wait Time:                  6208s     103.46m     1.72h    0.07d  0.000 y
# Average job time:                  30s       0.50m     0.01h    0.00d
# Longest job:                      289s       4.82m     0.08h    0.00d
# Submission to last job:          2800s      46.67m     0.78h    0.03d

    #	Third cluster run to convert lav's to axt's
    #	Does not work on kki since /scratch on the iservers is not the
    #	same as /scratch on the other clusters.
    ssh kk
    cd /cluster/data/hg17/bed/blastz.mm4
    /cluster/data/hg17/jkStuff/BlastZ_run2.sh
    cd run.2
    para try, check, push, etc ...
# Completed: 45 of 46 jobs
# Crashed: 1 jobs
# CPU time in finished jobs:       2389s      39.82m     0.66h    0.03d  0.000 y
# IO & Wait Time:                 13374s     222.90m     3.71h    0.15d  0.000 y
# Average job time:                 350s       5.84m     0.10h    0.00d
# Longest job:                     1426s      23.77m     0.40h    0.02d
# Submission to last job:          1440s      24.00m     0.40h    0.02d

    #	chr19 failing due to out of memory.  Run this job individually
    #	on kolossus, adjusting the location of the nib directories:
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.mm4
    sed -e "s/i386/x86_64/g" /cluster/bin/scripts/blastz-chromlav2axt > \
	x86_64-chromlav2axt
    chmod +x x86_64-chromlav2axt
    time ./x86_64-chromlav2axt \
	/cluster/data/hg17/bed/blastz.mm4/lav/chr19 \
	/cluster/data/hg17/bed/blastz.mm4/axtChrom/chr19.axt \
	/cluster/bluearc/scratch/hg/gs.18/build35/bothMaskedNibs \
	/cluster/bluearc/scratch/mus/mm4/softNib
    #	real    24m28.955s
    #	user    6m40.990s
    #	sys     1m16.500s

    # translate sorted axt files into psl
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm4
    mkdir -p pslChrom
    set tbl = "blastzMm4"
    foreach f (axtChrom/chr*.axt)
      set c=$f:t:r
      echo "Processing chr $c"
      /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
    end
    #	This takes more than an hour.  You can shorten this by changing
    #	that command to a simple echo, put the results into a file,
    #	split the file into four parts and run the four files as shell
    #	scripts on eieio to have four processes running at the same
    #	time.  Load on eieio gets up to about 20 which is reasonable.

    # Load database tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm4/pslChrom
    bash
    for F in chr*_blastzMm4.psl
    do
	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${F}
	echo "${F} done"
    done
    # this is a 55 minute job
    # exit bash if you are tcsh

    # featureBits on blastzMm3 or 4 will not work on hgwdev, runs out of
    # memory.  But if you reset your ~/.hg.conf to use the read-only
    #	user and contact the hgwdev host, then use the x86_64 featureBits
    # featureBits hg16 blastzMm4
    # 1056761609 bases of 2865248791 (36.882%) in intersection
    # featureBits hg17 blastzMm4
    # 1056201417 bases of 2866216770 (36.850%) in intersection

# CHAIN MM4 BLASTZ (DONE - 2004-06-29 - Hiram)
# redone with the 'axtFilter -notQ_random' removed - 2004-06-23

# The axtChain is best run on the small kluster, or the kk9 kluster
    ssh kk9
    mkdir -p /cluster/data/hg17/bed/blastz.mm4/axtChain/run1
    cd /cluster/data/hg17/bed/blastz.mm4/axtChain/run1
    mkdir out chain

    ls -1S /cluster/data/hg17/bed/blastz.mm4/axtChrom/*.axt > input.lst
    cat << '_EOF_' > gsub
#LOOP
doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} out/$(root1).out
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

#  May need -minScore=5000 for all chroms if chr19 won't finish on kolossus

    cat << '_EOF_' > doChain
#!/bin/csh
axtChain $1 /iscratch/i/gs.18/build35/bothMaskedNibs \
	/iscratch/i/mm4/softNib $2 > $3
'_EOF_'
    # << this line makes emacs coloring happy
    chmod a+x doChain

    # 46 jobs
    gensub2 input.lst single gsub jobList
    para create jobList
    para try
    para push # ... etc ...
# Completed: 45 of 46 jobs
# CPU time in finished jobs:       6575s     109.58m     1.83h    0.08d  0.000 y
# IO & Wait Time:                  9274s     154.57m     2.58h    0.11d  0.000 y
# Average job time:                 352s       5.87m     0.10h    0.00d
# Longest job:                     3121s      52.02m     0.87h    0.04d
# Submission to last job:          3121s      52.02m     0.87h    0.04d
    #	one job wouldn't finish due to memory usage
    #	run the chr19 job on kolossus, takes an hour, gets up to 4 Gb
    #	memory usage

    # now on the file server, sort chains
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm4/axtChain
    time chainMergeSort run1/chain/*.chain > all.chain
    #	real    17m17.639s
    #	user    9m54.240s
    #	sys     1m31.210s
    #	(1.9 Gb result file !)

    time chainSplit chain all.chain
    #	real    27m32.278s
    #	user    9m46.970s
    #	sys     2m45.960s

    # optionally: rm run1/chain/*.chain

    # Load chains into database
    # next machine
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm4/axtChain/chain
    foreach i (*.chain)
        set c = $i:r
        hgLoadChain hg17 ${c}_chainMm4 $i
        echo done $c
    end

    #	featureBits hg17 chainMm4
    #	2829135227 bases of 2866216770 (98.706%) in intersection
    #	featureBits hg16 chainMm4
    #	2828363353 bases of 2865248791 (98.713%) in intersection

# NET MM4 (DONE - 2004-06-29 - Hiram)
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm4/axtChain
    mkdir preNet
    cd chain
    foreach i (*.chain)
      echo preNetting $i
      /cluster/bin/i386/chainPreNet $i /cluster/data/hg17/chrom.sizes \
                        /cluster/data/mm4/chrom.sizes ../preNet/$i
    end

    cd ..
    mkdir n1
    cd preNet
    foreach i (*.chain)
      set n = $i:r.net
      echo primary netting $i
      /cluster/bin/i386/chainNet $i -minSpace=1 /cluster/data/hg17/chrom.sizes \
	/cluster/data/mm4/chrom.sizes ../n1/$n /dev/null
    end

    cd ..
    cat n1/*.net | /cluster/bin/i386/netSyntenic stdin hNoClass.net
    #	memory usage 2504171520, utime 19373 s/100, stime 5906

    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm4/axtChain
    time netClass hNoClass.net hg17 mm4 mouse.net \
	-tNewR=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInMouse \
	-qNewR=/cluster/bluearc/scratch/mus/mm4/linSpecRep.notInHuman
    #	real    19m33.421s
    #	user    10m37.130s
    #	sys     1m45.630s

    # If things look good do
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm4/axtChain
    rm -r n1 hNoClass.net
    # Make a 'syntenic' subset of these with
    time netFilter -syn mouse.net > mouseSyn.net
    #	real    13m24.885s
    #	user    7m37.100s
    #	sys     1m5.760s

    # Load the nets into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm4/axtChain
    netFilter -minGap=10 mouse.net |  hgLoadNet hg17 netMm4 stdin
    netFilter -minGap=10 mouseSyn.net | hgLoadNet hg17 syntenyNetMm4 stdin
    #	real    44m20.735s
    #	user    15m58.620s
    #	sys     1m58.720s
    # check results
    # featureBits hg17 netMm4
    #	2824272033 bases of 2866216770 (98.537%) in intersection
    # featureBits hg16 netMm4
    #	2823565051 bases of 2865248791 (98.545%) in intersection

    # featureBits hg17 syntenyNetMm4
    #	2785830955 bases of 2866216770 (97.195%) in intersection
    # featureBits hg16 syntenyNetMm4
    #	2786960572 bases of 2865248791 (97.268%) in intersection

    # Add entries for net and chain to mouse/hg17 trackDb

    # make net
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm4/axtChain
    mkdir mouseNet
    time netSplit mouse.net mouseNet
    #	real    12m1.478s
    #	user    8m35.050s
    #	sys     1m7.230s

    #	extract axt's from net, and convert to maf's (DONE - Kate - 2004-06-24)
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm4/axtChain
    mkdir ../axtNet ../mafNet
cat > makeMaf.csh << '_EOF_'
    foreach f (mouseNet/chr*.net)
        set c = $f:t:r
        echo "netToAxt: $c.net -> $c.axt"
        rm -f ../axtNet/$c.axt
        netToAxt mouseNet/$c.net chain/$c.chain \
	    /cluster/data/hg17/nib /cluster/data/mm4/nib stdout | \
	    axtSort stdin ../axtNet/$c.axt
        axtToMaf ../axtNet/$c.axt \
            /cluster/data/hg17/chrom.sizes /cluster/data/mm4/chrom.sizes \
            ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=mm4.
	echo "Complete: $c.net -> axtNet/$c.axt -> mafNet/$c.maf"
    end
'_EOF_'
# << for emacs
    csh makeMaf.csh >&! makeMaf.log &
    tail -100f makeMaf.log

    ssh hgwdev
    mkdir -p /cluster/data/hg17/bed/blastz.mm4/axtBest
    cd /cluster/data/hg17/bed/blastz.mm4/axtBest
    ln -s ../axtNet/chr*.axt .

    # copy net axt's to download area
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm4/axtNet
    mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/vsMm4/axtNet
    cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsMm4/axtNet
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsMm4/axtNet
    gzip *.axt
    # add README.txt file to dir (use previous assembly's copy as template)

    #  Convert those axt files to psl
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm4
    mkdir pslBest
    foreach a (axtBest/chr*.axt)
	set c=$a:t:r
	echo "processing $c.axt -> ${c}_blastzBestMm4.psl"
    /cluster/bin/i386/axtToPsl axtBest/${c}.axt \
	S1.len S2.len pslBest/${c}_blastzBestMm4.psl
	echo "Done: ${c}_blastzBestMm4.psl"
    end

    # Load tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm4/pslBest
    for I in chr*BestMm4.psl
    do
	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
	echo "done ${I}"
    done

     # check results
    # featureBits hg17 blastzBestMm4
    #	1017319919 bases of 2866216770 (35.493%) in intersection
    # featureBits hg16 blastzBestMm4
    #	996722004 bases of 2865248791 (34.787%) in intersection

    # Make /gbdb links and add them to the axtInfo table:
     mkdir -p /gbdb/hg17/axtBest/Mm4
     cd /gbdb/hg17/axtBest/Mm4
     ln -s /cluster/data/hg17/bed/blastz.mm4/axtNet/chr*.axt .
     cd /cluster/data/hg17/bed/blastz.mm4/axtNet
     rm -f axtInfoInserts.sql
     foreach f (/gbdb/hg17/axtBest/Mm4/chr*.axt)
       set chr=$f:t:r
       echo "INSERT INTO axtInfo (species, alignment, chrom, fileName) \
                VALUES ('mm4','Blastz Best in Genome','$chr','$f');" \
         >> axtInfoInserts.sql
     end
    hgsql hg17 < ~/kent/src/hg/lib/axtInfo.sql
    #	table axtInfo may already exist, ignore create error.
    hgsql hg17 < axtInfoInserts.sql

# MAKING MOUSE SYNTENY (DONE - 2004-06-29 - Hiram)

ssh hgwdev
mkdir /cluster/data/hg17/bed/syntenyMm4
cd /cluster/data/hg17/bed/syntenyMm4

# Copy all the needed scripts from /cluster/data/hg16/bed/syntenyMm3
cp -p /cluster/data/hg17/bed/syntenyRn3/*.pl .

./syntenicBest.pl -db=hg17 -table=blastzBestMm4
./smooth.pl
./joinsmallgaps.pl
./fillgap.pl -db=hg17 -table=blastzBestMm4
./synteny2bed.pl
#	The five commands above
#	real    220m16.227s
#	user    0m22.940s
#	sys     0m3.960s

#	Used to load this in syntenyMm4, but that type is misleading to
#	the table browser and fails the checkTableCoords check.
#	Better to use this ensRatMusHom type:
#	Need a new name here for the Mm4 to not conflict with Rn3
sed -e 's/ensPhusionBlast/ensRatMm4Hom/g' \
      $HOME/kent/src/hg/lib/ensPhusionBlast.sql \
      > ensRatMm4Hom.sql
hgLoadBed hg17 ensRatMm4Hom ucsc100k.bed -sqlTable=ensRatMm4Hom.sql
    #	featureBits hg17 ensRatMm4Hom
    #	2549307611 bases of 2866216770 (88.943%) in intersection
    #	featureBits hg16 syntenyMm4
    #	2560252977 bases of 2865248791 (89.355%) in intersection

# MAKING MOUSE AXTTIGHT FROM AXTBEST (DONE - 2004-06-29 - Hiram)
    # After creating axtBest alignments above, use subsetAxt to get axtTight:
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm4/axtNet
    mkdir -p ../axtTight
    foreach i (*.axt)
      echo $i
      subsetAxt  $i ../axtTight/$i \
        ~kent/src/hg/mouseStuff/subsetAxt/coding.mat 3400
    end

    # translate to psl
    cd ../axtTight
    mkdir ../pslTight
    foreach i (*.axt)
      set c = $i:r
      axtToPsl $i ../S1.len ../S2.len ../pslTight/${c}_blastzTightMm4.psl
      echo "Done: $i"
    end

    # Load tables into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm4/pslTight
    for I in chr*TightMm4.psl
    do
	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
	echo "done ${I}"
    done

    #	Compare results with previous assembly:
    #	featureBits hg17 blastzTightMm4
    #	166569246 bases of 2866216770 (5.811%) in intersection
    #	featureBits hg16 blastzTightMm4
    #	162641577 bases of 2865248791 (5.676%) in intersection

    # copy  axt's to download area
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm4/axtTight
    mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/vsMm4/axtTight
    cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsMm4/axtTight
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsMm4/axtTight
    gzip *.axt
    # add README.txt file to dir (use previous assembly's copy as template)

# BLASTZ MM4 CLEAN UP (DONE - 2004-07-02 - Hiram)
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm4
    nice rm -rf raw &
    nice rm axtChain/run1/chain/* &
    nice rm -fr axtChain/n1 axtChain/hNoClass.net &
    nice gzip axtChrom/* pslChrom/* lav/*/* axtChain/all.chain axtChain/*.net &


# BLASTZ CHIMP panTro1 (DONE 2004-06-22 kate)
# NOTE: Ran with abridge repeats=0, although SMSK was set
# Looked better than running with abridge=1, which had very
# chopped-up alignments

    ssh kk
    cd /cluster/data/hg17/bed
    mkdir -p blastz.panTro1.2004-06-22
    rm -f blastz.panTro1
    cd blastz.panTro1.2004-06-22

    cat << 'EOF' > DEF
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin

ALIGN=blastz-run
BLASTZ=blastz
BLASTZ_H=2000
BLASTZ_ABRIDGE_REPEATS=0

# Specific settings for chimp
BLASTZ_Y=3400
BLASTZ_T=2
BLASTZ_K=4500
BLASTZ_Q=/cluster/data/penn/human_chimp.q

# TARGET: Human
SEQ1_DIR=/scratch/hg/gs.18/build35/bothMaskedNibs
# not used 
SEQ1_RMSK=
# not used
SEQ1_FLAG=
SEQ1_SMSK=/iscratch/i/hg17/linSpecRep.chimp
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Chimp
SEQ2_DIR=/scratch/chimp/panTro1/nib
# not currently used
SEQ2_RMSK=/iscratch/i/chimp/panTro1/linSpecRep.human
# not currently used
SEQ2_FLAG=
SEQ2_SMSK=
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/blastz.panTro1.2004-06-22

DEF=$BASE/DEF
RAW=$BASE/raw
CDBDIR=$BASE
SEQ1_LEN=$BASE/S1.len
SEQ2_LEN=$BASE/S2.len

'EOF'
    # << this line keeps emacs coloring happy

    # first cluster run: raw blastz alignments
    ssh kk
    bash # if a csh/tcsh user
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22
    source DEF
    mkdir $RAW run.0
    /cluster/home/angie/hummus/make-joblist $DEF > $BASE/run.0/j
    sh ./xdir.sh
    cd run.0
    sed -e 's@^blastz-run@/cluster/bin/penn/blastz-run@' j > jobList
    para create jobList
        # 160270 jobs written to batc
    para try, check, push, check, ....
#CPU time in finished jobs:    2399227s   39987.11m   666.45h   27.77d  0.076 y
#IO & Wait Time:                503100s    8385.00m   139.75h    5.82d  0.016 y
#Average job time:                  18s       0.30m     0.01h    0.00d
#Longest running job:                0s       0.00m     0.00h    0.00d
#Longest finished job:            2073s      34.55m     0.58h    0.02d
#Submission to last job:         10843s     180.72m     3.01h    0.13d


    # second cluster run: lift raw alignments -> lav dir
    ssh kki
    bash # if a csh/tcsh user
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22
    source DEF
    mkdir run.1 lav
    /cluster/bin/scripts/blastz-make-out2lav $DEF $BASE > $BASE/run.1/jobList
    cd run.1
    wc -l jobList
    para create jobList
        # 341 jobs
    para try, check, push, etc ...
# CPU time in finished jobs:       3458s      57.63m     0.96h    0.04d  0.000 y
# IO & Wait Time:                 57996s     966.60m    16.11h    0.67d  0.002 y
# Average job time:                 180s       3.00m     0.05h    0.00d
# Longest job:                      483s       8.05m     0.13h    0.01d
# Submission to last job:          1498s      24.97m     0.42h    0.02d

    # third run: lav -> axt -> psl
    ssh kki
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22
    mkdir axtChrom pslChrom run.2
    cd run.2
    cat << '_EOF_' > do.csh
#!/bin/csh -ef
cd $1
set chr = $1:t
cat `ls -1 *.lav | sort -g` \
| /cluster/bin/x86_64/lavToAxt stdin \
    /iscratch/i/hg17/bothMaskedNibs /iscratch/i/chimp/panTro1/nib stdout \
| /cluster/bin/x86_64/axtSort stdin ../../axtChrom/$chr.axt 
/cluster/bin/x86_64/axtToPsl ../../axtChrom/$chr.axt ../../S1.len ../../S2.len \
  ../../pslChrom/$chr.psl
'_EOF_'
    # << this line keeps emacs coloring happy
    chmod a+x do.csh
    cp /dev/null jobList
    for d in ../lav/chr*; do
      echo "do.csh $d" >> jobList
    done
    para create jobList
        # 46 jobs
    para try, check, push, check
#Completed: 42 of 42 jobs
#Average job time:                  38s       0.64m     0.01h    0.00d
#Longest job:                      147s       2.45m     0.04h    0.00d
#Submission to last job:           147s       2.45m     0.04h    0.00d

    # Load database tables (takes an hour or so)
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/pslChrom
cat > load.csh << 'EOF'
    foreach f (chr*.psl)
	set table = $f:r_blastzPanTro1
	echo "loading ${table}"
	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 -table=$f:r_${table} $f
    end
'EOF'
# << for emacs
    csh load.csh >&! load.log & 
    tail -100f load.log


# CHAIN CHIMP BLASTZ (6/23/04 kate)
    # Run axtChain on little cluster
    # first copy input to bluearc, as eieo bogs down if even mini-cluster
    # gets input from it !?
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22
    cp -rp axtChrom /cluster/bluearc/hg17/blastz.panTro1.2004-06-22/axtChrom

    ssh kki
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22
    mkdir -p axtChain/run1
    cd axtChain/run1
    mkdir out chain
    ls -1S /cluster/bluearc/hg17/blastz.panTro1.2004-06-22/axtChrom/*.axt \
      > input.lst
    cat << '_EOF_' > gsub
#LOOP
doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    cat << '_EOF_' > doChain
#!/bin/csh -fe
set c = $1:r:t
axtChain $1 -scoreScheme=/cluster/data/blastz/human_chimp.q \
        /iscratch/i/hg17/bothMaskedNibs \
        /iscratch/i/chimp/panTro1/nib /tmp/$c.chain.$$ > /tmp/$c.out.$$
set ret = $status
mv -f /tmp/$c.chain.$$ $2
mv -f /tmp/$c.out.$$ $3
exit $status
'_EOF_'
    # << this line makes emacs coloring happy
    chmod a+x doChain
    gensub2 input.lst single gsub jobList
    para create jobList
    para try, check, push, check...
    # TODO
    rm -fr /cluster/bluearc/hg17/blastz.panTro1.2004-06-22/axtChrom
    echo "remove after 7/1/04" > /cluster/bluearc/hg17/blastz.panTro1.2004-06-22/axtChrom/README

    # now on the cluster server, sort chains
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
    chainMergeSort run1/chain/*.chain > all.chain
    chainSplit chain all.chain
    # TODO
    rm run1/chain/*.chain
    echo "remove after 7/1/04" > run1/chain/README

    # Load chains into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain/chain
    foreach i (*.chain)
        set c = $i:r
        echo loading $c
        hgLoadChain hg17 ${c}_chainPanTro1 $i
    end
    # TODO
    featureBits hg16 chainPanTro1Link
        #2627280557 bases of 2865248791 (91.695%) in intersection
    featureBits hg17 chainPanTro1Link
        # 2633869032 bases of 2866216770 (91.894%) in intersection


# NET CHIMP (DONE 2004-6-24 kate)
    # Redone to make chimp.net on 2004-10-11 kate (other files have
    # new times, but are the same as 6-24 versions)
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
    #chainPreNet all.chain ../S1.len ../S2.len stdout \
    #| chainNet stdin -minSpace=1 ../S1.len ../S2.len stdout /dev/null \
    #| netSyntenic stdin noClass.net

    time chainPreNet all.chain ../S1.len ../S2.len stdout \
    | chainNet stdin -minSpace=10 ../S1.len ../S2.len human.net chimp.net
        # 42.860u 2.080s 2:11.11 34.2% 
    netSyntenic human.net noClass.net

    # Add classification info using db tables:
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
    netClass noClass.net hg17 panTro1 human.net
    rm noClass.net

    # Make a 'syntenic' subset:
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
    # TODO
    #rm noClass.net
    # Make a 'syntenic' subset of these with
    # NOTE: we used -chimpSyn filtering for the reciprocal best nets
    # on hg16 -- perhaps should use for nets here as well
    netFilter -chimpSyn human.net > humanSyn.net

    # Load the nets into database 
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
    netFilter -minGap=10 human.net |  hgLoadNet hg17 netPanTro1 stdin
    netFilter -minGap=10 humanSyn.net | hgLoadNet hg17 netSyntenyPanTro1 stdin
    # Add entries for chainPanTro1, netPanTro1 to
    # human/hg17 trackDb

    # save chimp net to downloads area
    ssh eieio
    cd /cluster/data/hg17/blastz.panTro1/axtChain
    nice gzip chimp.net
    cp chimp.net.gz /usr/local/apache/htdocs/goldenPath/panTro1/vsHg17
    cd /usr/local/apache/htdocs/goldenPath/panTro1/vsHg17
    md5sum *.gz > md5sum.txt


# RECIPROCAL BEST CHAINS FOR ENSEMBL GENE BUILD (DONE 2004-10-11 kate)

    # Starting with the chimp-reference net, which contains the best human
    # alignments to chimp, extract the subset of chains in the net.
    # (these are the "best" chains of human alignments to chimp).
    # Net these chains and use the resulting human-reference net (the
    # "reciprocal best" net).  Extract the chains from this net to
    # obtain "reciprocal best" chains of chimp alignments to human.

    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
    mkdir rBest
    grep chain all.chain | wc -l

    # extract "best" chains from the chimp-reference net
    time chainSwap all.chain stdout | \
        netChainSubset chimp.net stdin stdout | \
        chainSort stdin rBest/chimp.best.chain
    grep chain rBest/chimp.best.chain | wc -l
        # 64396 

    # for comparison later, extract "best" chains from human-reference net
    netChainSubset human.net all.chain stdout | \
        chainSort stdin rBest/human.best.chain
    cd rBest

    # net the best chains from the chimp net and pull the human-ref net
    # (Daryl accidentally deleted human.rbest.net and rebuilt it with the 
    #  same command on 8/14/2005, resulting in a file of the same size)
    time chainPreNet chimp.best.chain ../../S2.len ../../S1.len stdout | \
       chainNet stdin -minSpace=10 ../../S2.len ../../S1.len \
                         /dev/null human.rbest.net

    # extract "reciprocal best" chains from the "best" human-reference net
    netChainSubset human.rbest.net ../all.chain stdout | \
        chainSort stdin human.rbest.chain

    # take a look
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
    cd rBest
    mkdir rBestChain
    chainSplit rBestChain human.rbest.chain
    hgLoadChain hg17 chr7_rBestChainPanTro1 rBestChain/chr7.chain
        # Loading 1639 chains into hg17.chr7_rBestChainPanTro1
    mkdir bestChain
    chainSplit bestChain human.best.chain
    hgLoadChain hg17 chr7_bestChainPanTro1 bestChain/chr7.chain
        # Loading 6516 chains into hg17.chr7_bestChainPanTro1
    # compare 
    hgsql hg16 -s -e "select count(*) from chr7_rBestChainPanTro1"
        # 2416
    # spot-checked by comparing chr7 best and rbest:
    # 1. for a a chain appearing in rBest,  click thru to human browser, 
    #   then via chimp net back to human browser at same region
    # 2. for a chain in "best", but not rBest, do the same, verify
    #   that it produces a different region in the human browser

    # post pre-Q/A file for ensembl download
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain/rBest
    gzip human.rbest.chain
    cp human.rbest.chain.gz \
        /usr/local/apache/htdocs/kate/ensembl/hg17-panTro1.rbest.chain.gz
    cd /usr/local/apache/htdocs/kate/ensembl
    md5sum *.gz > md5sum.txt
    mv hg17-panTro1.rbest.chain.gz /usr/local/apache/htdocs/hg17/vsPanTro1/hg17.panTro1.rbest.chain.gz

    # save as reciprocal best liftover chain (2005-02-22 kate)
    gunzip -c  human.rbest.chain.gz > \
        /cluster/data/hg17/bed/liftOver/hg17ToPanTro1.rbest.chain

    # cleanup (TODO -- after QA)
    ssh hgwdev
    hgsql hg17 -e "drop table chr7_rBestChainPanTro1"
    hgsql hg17 -e "drop table chr7_bestChainPanTro1"
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
    mv rBest/human.rbest.chain.gz ..
    rm -fr rBest

# RECIPROCAL BEST AXT'S FROM RECIPROCAL BEST CHAIN (2005-08-16 kate)
#       (requested by Daryl)
    cd /cluster/data/hg17/bed/blastz.panTro1
    mkdir -p axtRBestNet
cat > makeRbestAxt.csh << 'EOF'
foreach f (axtChain/rBest/rBestChain/*.chain)
    set c = $f:t:t:r
    echo $c
    chainToAxt $f /cluster/data/hg17/nib  /cluster/data/panTro1/nib stdout \
        | axtSort stdin axtRBestNet/$c.axt
    end
'EOF'
# << for emacs
    csh makeRbestAxt.csh >&! makeRbestAxt.log &


# GENERATE CHIMP MAF FOR MULTIZ FROM NET (DONE 2004-06-24 kate)
    # Redo to fix overlap problem using 8/05 netToAxt (2005-08-16 kate)
    # Replace bad chr5 axtNet and mafNet (2006-01-05 kate) 
    #   There was apparently a bad chr5 nib for a while...
    ssh kkstore02
    cd /cluster/data/hg17/bed/blastz.panTro1.2004-06-22/axtChain
    netSplit human.net net

    mkdir axtNet mafNet
cat > makeMaf.csh << 'EOF'
foreach f (axtChain/net/*.net)
    set c = $f:t:r
    netToAxt $f axtChain/chain/$c.chain /cluster/data/hg17/nib \
        /cluster/data/panTro1/nib stdout | axtSort stdin axtNet/$c.axt
    axtToMaf axtNet/$c.axt  \
        /cluster/data/hg17/chrom.sizes /cluster/data/panTro1/chrom.sizes \
        mafNet/$c.maf -tPrefix=hg17. -qPrefix=panTro1.
    end
'EOF'
# << for emacs
    csh makeMaf.csh >&! makeMaf.log &
    tail -100f makeMaf.log

    mkdir -p /cluster/bluearc/hg17/mafNet
    cp -rp mafNet /cluster/bluearc/hg17/mafNet/panTro1


# MAKE PANTRO1 DOWNLOADABLES (DONE 2004-09-14 kate)
#  Redo panTro1.net.gz (it was truncated) 2004-10-07 kate
# Redo axtNets with non-overlapped versions (2005-08-29 kate)
# Replace bad chr5 axtNet and mafNet (2006-01-05 kate) 
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.panTro1
    # gzip chains and nets
    mkdir gz
    cd gz
    nice gzip -c ../axtChain/all.chain  > panTro1.chain.gz
    nice gzip -c ../axtChain/human.net > panTro1.net.gz
    wc -l *.gz
    cd ../axtNet
    time nice gzip *.axt
        # 46 mins.
   
    ssh hgwdev
    # copy chains and nets to downloads area
    cd /usr/local/apache/htdocs/goldenPath/hg17
    mkdir -p vsPanTro1
    cd vsPanTro1
    mv /cluster/data/hg17/bed/blastz.panTro1/gz/*.gz .
    md5sum *.gz > md5sum.txt
    # copy in README and edit
    rmdir /cluster/data/hg17/bed/blastz.panTro1/gz
    mkdir -p axtNet
    cd axtNet
    cp /cluster/data/hg17/bed/blastz.panTro1/axtNet/*.axt.gz .
    md5sum *.gz > md5sum.txt


# RESCORE CHICKEN BLASTZ (DONE 6/23/04 angie)
    # Webb noticed low scores when using non-default BLASTZ_Q scoring matrix 
    # and repeats abridged --
    # PSU's restore_rpts program rescored alignments with default matrix 
    # instead of BLASTZ_Q matrix.  Rescore them here so the chainer sees 
    # the higher scores:
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14
    mkdir axtChrom.rescore
    foreach f (axtChrom/chr*.axt)
      axtRescore -scoreScheme=/cluster/data/blastz/HoxD55.q \
        $f axtChrom.rescore/$f:t
    end
    mv axtChrom axtChrom.preRescore
    mv axtChrom.rescore axtChrom


# CHAIN CHICKEN BLASTZ (DONE 6/23/04 angie)
    # Run axtChain on little cluster
    ssh kki
    cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14
    mkdir -p axtChain/run1
    cd axtChain/run1
    mkdir out chain
    ls -1S /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChrom/*.axt \
      > input.lst
    cat << '_EOF_' > gsub
#LOOP
doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    cat << '_EOF_' > doChain
#!/bin/csh
axtChain -scoreScheme=/cluster/data/blastz/HoxD55.q \
         -linearGap=/cluster/data/blastz/chickenHumanTuned.gap \
         -minScore=5000 $1 \
    /iscratch/i/hg17/bothMaskedNibs \
    /iscratch/i/galGal2/nib $2 > $3
'_EOF_'
    # << this line makes emacs coloring happy
    chmod a+x doChain
    gensub2 input.lst single gsub jobList
    para create jobList
    para try, check, push, check...
    # axtChrom/chr18_random.axt is empty, so the {out line +} check failed:
#Completed: 45 of 46 jobs
#Crashed: 1 jobs
#Average job time:                  46s       0.76m     0.01h    0.00d
#Longest job:                      273s       4.55m     0.08h    0.00d
#Submission to last job:           519s       8.65m     0.14h    0.01d

    # now on the cluster server, sort chains
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
    chainMergeSort run1/chain/*.chain > all.chain
    chainSplit chain all.chain
    rm run1/chain/*.chain

    # Load chains into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain/chain
    foreach i (*.chain)
        set c = $i:r
        echo loading $c
        hgLoadChain hg17 ${c}_chainGalGal2 $i
    end


# NET CHICKEN BLASTZ (DONE 6/23/04 angie)
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
    chainPreNet all.chain ../S1.len ../S2.len stdout \
    | chainNet stdin -minSpace=1 ../S1.len ../S2.len stdout /dev/null \
    | netSyntenic stdin noClass.net

    # Add classification info using db tables:
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
    netClass noClass.net hg17 galGal2 human.net

    # Make a 'syntenic' subset:
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
    rm noClass.net
    # Make a 'syntenic' subset of these with
    netFilter -syn human.net > humanSyn.net

    # Load the nets into database 
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
    netFilter -minGap=10 human.net |  hgLoadNet hg17 netGalGal2 stdin
    netFilter -minGap=10 humanSyn.net | hgLoadNet hg17 netSyntenyGalGal2 stdin
    # Add entries for chainGalGal2, netGalGal2, syntenyGalGal2 to 
    # human/hg17 trackDb

# XENOPUS BLASTZ/CHAIN/NET (DONE 9/24/04 jk)
# see makeXenTro1.doc and search for zb.hg17
# The results of this are also symlinked under hg17/bed

# GENERATE GALGAL2 MAF FOR MULTIZ FROM NET (DONE 6/23/04 angie)
#       Redo net axt's and maf's to fix overlap problem (use 8/05 netToAxt)
#               (2005-08-16 kate)
    ssh kkstore02
    cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
    netSplit human.net net
    cd ..
    mkdir axtNet mafNet
cat > makeMaf.csh << 'EOF'
    foreach f (axtChain/net/*)
      set chr = $f:t:r
      netToAxt $f axtChain/chain/$chr.chain /cluster/data/hg17/nib \
        /cluster/data/galGal2/nib stdout \
      | axtSort stdin axtNet/$chr.axt
      axtToMaf axtNet/$chr.axt \
            /cluster/data/hg17/chrom.sizes /cluster/data/galGal2/chrom.sizes \
            mafNet/$chr.maf -tPrefix=hg17. -qPrefix=galGal2.
    end
'EOF'
# << for emacs
    csh makeMaf.csh >&! makeMaf.log &
    mkdir -p /cluster/bluearc/hg17/mafNet
    cp -rp mafNet /cluster/bluearc/hg17/mafNet/galGal2


# MAKE VSGALGAL2 DOWNLOADABLES (REDONE 9/13/04 angie)
#   REDO axtNet's to fix overlaps (2005-09-12 kate)
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain
    gzip -c all.chain > /cluster/data/hg17/zip/chicken.chain.gz
    gzip -c human.net > /cluster/data/hg17/zip/chicken.net.gz
    mkdir /cluster/data/hg17/zip/axtNet
    foreach f (axtNet/chr*axt)
      gzip -c $f > /cluster/data/hg17/zip/$f.gz
    end
    # Doh! above for loop didn't work because all axt's have been removed 
    # from this dir!  :|  Just this once, regenerate compressed axtNet on 
    # the fly:
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.galGal2.2004-06-14/axtChain/net
    foreach f (*.net)
      set chr = $f:t:r
      echo $chr
      netToAxt $f ../chain/$chr.chain /cluster/data/hg17/nib \
        /cluster/data/galGal2/nib stdout \
      | axtSort stdin stdout \
      | gzip -c > /cluster/data/hg17/zip/axtNet/$chr.axt.gz
    end    

    ssh hgwdev
    mkdir /usr/local/apache/htdocs/goldenPath/hg17/vsGalGal2
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsGalGal2
    mv /cluster/data/hg17/zip/chicken*.gz .
    mv /cluster/data/hg17/zip/axtNet .
    md5sum *.gz */*.gz > md5sum.txt
    # Copy over & edit README.txt w/pointers to chain, net formats.

    # REDO axtNet downloads to fix overlaps (2005-09-13 kate)
    ssh kkstore02
    cd /cluster/data/hg17/bed/blastz.galGal2/axtNet
    nice gzip *.axt
    md5sum *.axt.gz > md5sum.txt

    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsGalGal2
    mv axtNet axtNet.old
    ln -s /cluster/data/hg17/bed/blastz.galGal2/axtNet .


# 8-WAY MULTIZ MULTIPLE ALIGNMENT WITH MM5 (DONE 2004-07-13 kate)
#       Redo, below to fix overlapping alignments (2005-08-16 kate)

    ssh eieio
    set multizDir = multiz.2004-07-13
    set workingDir = /cluster/bluearc/hg17/$multizDir
    ln -s $workingDir /cluster/bluearc/hg17/multiz8way
    mkdir -p $workingDir
    mkdir -p /cluster/data/hg17/bed/$multizDir
    cd /cluster/data/hg17/bed/$multizDir

# wrapper script for multiz
    # NOTE: first arg is pairwise, 2nd arg is multiple (to add to) 
    # NOTE: next time, modify script so it only needs one arg -- saves the
    # multiple dirname in a file for use by the next run
    cat << 'EOF' > doMultiz.csh
#!/bin/csh -fe
mkdir -p $3:h
/cluster/bin/penn/multiz $1 $2 - > $3
'EOF'
# << for emacs
    cat << 'EOF' > gsub
#LOOP
../doMultiz.csh {check in line /cluster/bluearc/hg17/multiz.2004-07-13/$(dir1)/$(root2).maf} {check in line /cluster/bluearc/hg17/multiz.2004-07-13/$(root1)/$(root2).maf} {check out line+ /cluster/bluearc/hg17/multiz.2004-07-13/$(root1)$(dir1)/$(root2).maf}
#ENDLOOP
'EOF'
# << for emacs
    chmod +x doMultiz.csh

    ssh eieio
    set workingDir = /cluster/bluearc/hg17/multiz.2004-07-13

    # copy mafs to bluearc -- chimp
    mkdir $workingDir/panTro1
    cp /cluster/data/hg17/bed/blastz.panTro1/mafNet/*.maf \
                $workingDir/panTro1
    ls $workingDir/panTro1/*.maf > chrom.lst

    # mouse
    mkdir $workingDir/mm5
    cp /cluster/data/hg17/bed/blastz.mm5/mafNet/chr*.maf $workingDir/mm5

    # rat
    mkdir $workingDir/rn3
    cp /cluster/data/hg17/bed/blastz.rn3/mafNet/chr*.maf $workingDir/rn3

    # dog
    mkdir $workingDir/canFam1
    foreach f (/cluster/data/hg17/bed/blastz.canFam1.2004-07-08/mafNet/chr*.maf)
        set c = $f:r:r:t
        echo $c
        cp $f $workingDir/canFam1/$c.maf
    end

    # chicken
    mkdir $workingDir/galGal2
    foreach f (/cluster/data/hg17/bed/blastz.galGal2/mafNet/chr*.maf)
        set c = $f:r:r:t
        cp $f $workingDir/galGal2/$c.maf
    end

    # fugu
    mkdir $workingDir/fr1
    cp /cluster/data/hg17/bed/blastz.fr1/mafNet/chr*.maf $workingDir/fr1

    # zebrafish
    mkdir $workingDir/danRer1
    cp /cluster/data/hg17/bed/blastz.danRer1.swap/mafNet/chr*.maf \
                        $workingDir/danRer1

    # first multiz - add in mm5 mouse to human/chimp
    # 
    ssh kki
    set multizDir = multiz.2004-07-13
    set workingDir = /cluster/bluearc/hg17/$multizDir
    cd /cluster/data/hg17/bed/$multizDir
    mkdir run.mm5
    cd run.mm5
    echo "mm5/panTro1" > species.lst
    gensub2 species.lst ../chrom.lst ../gsub jobList
    para create jobList
        # 46 jobs
    para try, check, push, check
# CPU time in finished jobs:       6620s     110.33m     1.84h    0.08d  0.000 y
# IO & Wait Time:                  3685s      61.42m     1.02h    0.04d  0.000 y
# Average job time:                 224s       3.73m     0.06h    0.00d
# Longest job:                      819s      13.65m     0.23h    0.01d
# Submission to last job:          1474s      24.57m     0.41h    0.02d
    cd ..

    # rat
    mkdir run.rn3
    cd run.rn3
    echo "rn3/panTro1mm5" > species.lst
    gensub2 species.lst ../chrom.lst ../gsub jobList
    para create jobList
        # 46 jobs
    para try, check, push, check
    cd ..

    # dog
    mkdir run.canFam1
    cd run.canFam1
    echo "canFam1/panTro1mm5rn3" > species.lst
    gensub2 species.lst ../chrom.lst ../gsub jobList
    para create jobList
        # 46 jobs
    para try, check, push, check
    cd ../

    # chicken
    mkdir run.galGal2
    cd run.galGal2
    echo "galGal2/panTro1mm5rn3canFam1" > species.lst
    gensub2 species.lst ../chrom.lst ../gsub jobList
    # no alignment file for chr18_random -- create one so we can create jobList
    touch $workingDir/galGal2/chr18_random.maf
    para create jobList
        # 46 jobs
    para try, check, push, check
    # 1 crashed job for empty file chr18_random
    cd ..

    # fugu
    mkdir run.fr1
    cd run.fr1
    echo "fr1/panTro1mm5rn3canFam1galGal2" > species.lst
    gensub2 species.lst ../chrom.lst ../gsub jobList
    # create empty alignment file for missing one (no alignments)
    touch /cluster/bluearc/hg17/multiz.2004-07-13/fr1/chr6_hla_hap1.maf
    para create jobList
        # 46 jobs
    para try, check, push, check
    # 1 crashed job for empty file chr6_hla_hap1
    cd ..

    # zebrafish
    mkdir run.danRer1
    cd run.danRer1
    echo "danRer1/panTro1mm5rn3canFam1galGal2fr1" > species.lst
    gensub2 species.lst ../chrom.lst ../gsub jobList
    para create jobList
        # 46 jobs
    para try, check, push, check
    cd ..

    # copy 8-way mafs to build directory
    ssh eieio
    set multizDir = multiz.2004-07-13
    set workingDir = /cluster/bluearc/hg17/$multizDir
    ln -s $workingDir/panTro1mm5rn3canFam1galGal2fr1danRer1 $workingDir/maf
    cd /cluster/data/hg17/bed/multiz.2004-07-13
    mkdir maf
    cp $workingDir/maf/*.maf maf

    # copy to download area (2004-07-27 angie)
    # moved gzipped files to mafDownload dir and recreated symlinks 
    #   (2006-04-23 kate)
    cd /usr/local/apache/htdocs/goldenPath/hg17
    mkdir -p mzPt1Mm5Rn3Cf1Gg2Fr1Dr1
    # gzipped & copied maf files from /cluster/data/hg17/bed/multiz8way/maf
    # dumped table and gzipped for download (user request to after file
    # removed when the track was replaced by 18way).
    cd /cluster/data/hg17/bed/multiz8way/mafDownloads
    hgsqldump --all -c --tab=. hg17 multiz8way
    ssh kkstore02 \
        'gzip /cluster/data/hg17/bed/multiz8way/mafDownloads/multiz8way.{sql,txt}'
    ln -s /cluster/data/hg17/bed/multiz8way/mafDownloads/multiz8way.{sql,txt}.gz \
        /usr/local/apache/htdocs/goldenPath/hg17/multiz8way

    # load summary table (2005-09-27)
    cd /cluster/data/hg17/bed/multiz.2004-07-13/maf
    time cat chr*.maf | hgLoadMafSummary hg17 multiz8waySummary stdin
    # 30 minutes ?
    # NOTE: this didn't improve track display time at 5MB, so
    #   I'm leaving out of trackDb (sticking with pairwise maf's) for now
    #   It may be that this helps performance only with larger numbers
    #   of species.

    # Create upstream files for download (2004-09-13 kate)
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz8way
    echo hg17 panTro1 mm5 rn3 canFam1 galGal2 fr1 danRer1 > org.txt
    # mafFrags takes a while
    foreach i (1000 2000 5000)
        echo "making upstream$i.maf"
        featureBits hg17 refGene:upstream:$i -fa=/dev/null -bed=up.bad
        awk -F '\t' '{printf("%s\t%s\t%s\t%s\t%s\t%s\n", $1, $2, $3, substr($4, 0, 9), 0, $5)}' up.bad > up.bed
        rm up.bad
        mafFrags hg17 multiz8way up.bed upstream$i.maf -orgs=org.txt
        rm up.bed
    end
    ssh eieio
    cd /cluster/data/hg17/bed/multiz8way
    nice gzip upstream{1000,2000,5000}.maf
        # 6 mins.
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17
    ln -s mzPt1Mm5Rn3Cf1Gg2Fr1Dr1 multiz8way
    mv /cluster/data/hg17/bed/multiz8way/upstream*.maf.gz multiz8way



# PHYLO-HMM (PHASTCONS) CONSERVATION FOR 8-WAY WITH MM5 (DONE 2004-07-20 kate)
# (this was partially redone by acs using the new phastCons, 08-28;
# I've tried to merge the two sets of docs into one cohesive
# description)
# More revisions, acs, 09-13

    ssh eieio
    set path = ($path /cluster/bin/phast)
    cd /cluster/data/hg17/bed/multiz.2004-07-13
    mkdir cons
    cd cons

    #break up the genome-wide MAFs into pieces
    mkdir /cluster/bluearc/hg17/chrom
    cd /cluster/data/hg17
    foreach f (`cat chrom.lst`)
        echo $f
        cp $f/*.fa /cluster/bluearc/hg17/chrom
    end

    ssh kki
    cd /cluster/data/hg17/bed/multiz.2004-07-13/cons
    mkdir run.split
    cd run.split
    set WINDOWS = /cluster/bluearc/hg17/multiz.2004-07-13/cons/WINDOWS
    rm -fr $WINDOWS
    mkdir -p $WINDOWS
    cat << 'EOF' > doSplit.sh
#!/bin/sh

PHAST=/cluster/bin/phast
FA_SRC=/cluster/bluearc/hg17/chrom
WINDOWS=/cluster/bluearc/hg17/multiz.2004-07-13/cons/WINDOWS

maf=$1
c=`basename $maf .maf`
echo $c
mkdir -p /scratch/msa_split
${PHAST}/msa_split $maf -i MAF -M ${FA_SRC}/$c.fa -O hg17,panTro1,mm5,rn3,canFam1,galGal2,fr1,danRer1 -w 1000000,0 -r /scratch/msa_split/$c -o SS -I 1000 -B 5000
[ $? -eq 0 ] || exit 1
echo "Copying..."
cd /scratch/msa_split
for file in $c.*.ss ; do gzip -c $file > ${WINDOWS}/$file.gz ; done
[ $? -eq 0 ] || exit 1
rm -f /scratch/msa_split/$c.*.ss
echo "Done copying"
echo "Done" >> ${WINDOWS}/$c.done
'EOF'
# << for emacs
    chmod +x doSplit.sh
    rm -f jobList
    foreach file (/cluster/bluearc/hg17/multiz.2004-07-13/maf/*.maf) 
        set c = $file:t:r
	echo "doSplit.sh $file {check out line+ $WINDOWS/$c.done}" >> jobList
    end
    
    para create jobList
        # 46 jobs
    para try
    para check
    para push
        # 2 crashed jobs -- due to no alignments in input maf
        # chr18_random, chr6_hla_hap1
    cd ..

    # now generate conservation scores and predicted elements
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz.2004-07-13/cons
    mkdir run.elements   
    # despite the name, I've put the elements and the new conservation
    # scores here

    # first produce a rough starting model; in this case, we can just
    # use the model previously estimated (see the entry below on PHYLOFIT/PHASTCONS)
    cp /cluster/bluearc/hg17/multiz.2004-07-13/panTro1mm5rn3canFam1/hg17panTro1rn3mm5canFam1galGal2fr1danRer1.mod starting-tree.mod
    # In other cases, it would be sufficient to choose an arbitrary
    # input file from the WINDOWS directory (choose one with plenty of
    # data, i.e., large NTUPLES) and run phyloFit on it with the
    # correct tree topology, e.g.,
    # phyloFit -i SS datafile.ss --tree \
    #    "(((((hg17,panTro1),(mm5,rn3)),canFam1),galGal2),(fr1,danRer1))" \
    #    --out-root starting-tree

    # Get genome-wide average GC content (for all species together,
    # not just the reference genome).  If you have a globally
    # estimated tree model, as above, you can get this from the
    # BACKGROUND line in the .mod file.  E.g.,
# ALPHABET: A C G T
# ...
# BACKGROUND: 0.294633 0.205082 0.205189 0.295097
    # This implies a GC content of 0.205 + 0.205 = 0.410

    # If you do *not* have a global tree model and you do not know
    # your GC content, you can get it directly from the MAFs with
    # a command like:
    #	msa_view --aggregate hg17,panTro1,rn3,mm5,canFam1,galGal2,danRer1,fr1 \
    #       -i MAF --summary-only /cluster/data/hg17/bed/multiz.2004-07-13/maf/chr*.maf\
    #	    > maf_summary.txt
    # This will take a little while (30-60 min).  Run on eieio. 

    # now set up cluster job to estimate model parameters.  Parameters
    # will be estimated separately for each alignment fragment then
    # will be combined across fragments
    cat << 'EOF' > doEstimate.sh
#!/bin/sh
zcat $1 | /cluster/bin/phast/phastCons - starting-tree.mod --gc 0.410 --nrates 1,1 --no-post-probs --ignore-missing --expected-lengths 12 --target-coverage 0.17 --quiet --log $2 --estimate-trees $3
EOF
    # Be sure to substitute in the right G+C content. Also, notice the
    # target coverage of 0.17.  We actually want 5% coverage here but
    # the final (posterior) coverage is only indirectly related to the
    # expected (prior) coverage.  One thing to consider is that we
    # only have about 40% alignment coverage (excluding chimp, which
    # doesn't help us much in identifying conserved regions).  As far
    # as phastCons is concerned, we want to aim for about 0.05 / 0.4 =
    # 0.125 coverage.  In this case, though, --target-coverage
    # 0.125 resulted in only about 4.1% coverage.  I had to iterate
    # a couple of times (using only chromosome 1) to find a value that
    # got me close to the target of 5%

    chmod u+x doEstimate.sh

    rm -fr LOG TREES
    mkdir -p LOG TREES
    rm -f jobs.lst
    # watch out: bash assumed below in a few places
    for f in /cluster/bluearc/hg17/multiz.2004-07-13/cons/WINDOWS/*.ss.gz ; do \
	root=`basename $f .ss.gz` ;\
	echo doEstimate.sh $f LOG/$root.log TREES/$root >> jobs.lst ;\
    done

    # run cluster job
    ssh kk, cd /cluster/data/hg17/bed/multiz.2004-07-13/cons/run.elements, para create, ...
    # takes about an hour

    # Now combine parameter estimates.  We can average the .mod files
    # using phyloBoot.  This must be done separately for the conserved
    # and nonconserved models
    ls TREES/*.cons.mod > cons.txt
    phyloBoot --read-mods '*cons.txt' --output-average ave.cons.mod > cons_summary.txt
    ls TREES/*.noncons.mod > noncons.txt
    phyloBoot --read-mods '*noncons.txt' --output-average ave.noncons.mod > noncons_summary.txt
    # look over the files cons_summary.txt and noncons_summary.txt.
    # The means and medians should be roughly equal and the stdevs
    # should be reasonably small compared to the means, particularly
    # for rate matrix parameters (at bottom) and for branches to the
    # leaves of the tree.  The stdevs may be fairly high for branches
    # near the root of the tree; that's okay.  Some min values may be
    # 0 for some parameters.  That's okay, but watch out for very large
    # values in the max column, which might skew the mean.  If you see
    # any signs of bad outliers, you may have to track down the
    # responsible .mod files and throw them out.  I've never had to do
    # this; the estimates generally seem pretty well behaved.

    # NOTE: Actually, a random sample of several hundred to a thousand
    # alignment fragments (say, a number equal to the number of
    # available cluster nodes) should be more than adequate for
    # parameter estimation.  If pressed for time, use this strategy.

    # Now we are ready to set up the cluster job for computing the
    # conservation scores and predicted elements.  It's all downhill
    # from here.

    cat << 'EOF' > doPhastCons.sh
#!/bin/sh

mkdir -p /cluster/bluearc/hg17/phastCons/POSTPROBS /cluster/bluearc/hg17/phastCons/ELEMENTS
pref=`basename $1 .ss.gz`
chr=`echo $pref | awk -F\. '{print $1}'`
tmpfile=/scratch/phastCons.$$
zcat $1 | /cluster/bin/phast/phastCons - ave.cons.mod,ave.noncons.mod --expected-lengths 12 --target-coverage 0.17 --quiet --seqname $chr --idpref $pref --viterbi /cluster/bluearc/hg17/phastCons/ELEMENTS/$pref.bed --score --require-informative 0 > $tmpfile
gzip -c $tmpfile > /cluster/bluearc/hg17/phastCons/POSTPROBS/$pref.pp.gz
rm $tmpfile
EOF
    chmod u+x doPhastCons.sh

    rm -fr /cluster/bluearc/hg17/phastCons/POSTPROBS /cluster/bluearc/hg17/phastCons/ELEMENTS
    rm -f jobs2.lst
    for f in /cluster/bluearc/hg17/multiz.2004-07-13/cons/WINDOWS/*.ss.gz ; do echo doPhastCons.sh $f >> jobs2.lst ; done

    # run cluster job
    ssh kk, cd /cluster/data/hg17/bed/multiz.2004-07-13/cons/run.elements, para create, ...
    logout
    # takes about 20 minutes

    # combine predictions and transform scores to be in 0-1000 interval
    # do in a way that avoids limits on numbers of args
    find /cluster/bluearc/hg17/phastCons/ELEMENTS -name "*.bed" > files
    rm -f splitfiles* all.raw.bed
    split files splitfiles
    for s in splitfiles* ; do awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", $1, $2, $3, $5, $5;}' `cat $s` >> all.raw.bed ; done
    /cluster/bin/scripts/lodToBedScore all.raw.bed > all.bed
    rm files splitfiles* 

    hgLoadBed hg17 phastConsElements all.bed
    hgLoadBed -chrom=chr1 hg17 phastConsElements all.bed

    # check coverage
    featureBits hg17 phastConsElements
#137850739 bases of 2866216770 (4.810%) in intersection
    # This should be close enough.  If necessary, you can rerun the
    # steps above with a different target coverage.  When hitting the
    # target is important, you may want to perform several iterations
    # using a representative subset of the entire dataset (human chr1
    # seems to work pretty well)

    # set up wiggle
    mkdir -p /cluster/bluearc/hg17/phastCons/wib
    cat << 'EOF' > doWigAsciiToBinary.sh
#!/bin/sh
chr=$1
zcat `ls /cluster/bluearc/hg17/phastCons/POSTPROBS/$chr.*.pp.gz | sort -t\. -k2,2n` | wigAsciiToBinary -chrom=$chr -wibFile=/cluster/bluearc/hg17/phastCons/wib/${chr}_phastCons stdin 
EOF
    chmod u+x doWigAsciiToBinary.sh

    rm -f jobs3.lst
    for chr in `ls /cluster/bluearc/hg17/phastCons/POSTPROBS | awk -F\. '{print $1}' | sort -u` ; do echo doWigAsciiToBinary.sh $chr >> jobs3.lst ; done

    # run a little wigAsciiToBinary cluster job
    ssh kk, etc.

    # copy wibs and wigs from bluearc
    rsync -av /cluster/bluearc/hg17/phastCons/wib .

    # load track
    hgLoadWiggle hg17 phastCons -pathPrefix=/gbdb/hg17/phastCons/wib \
                wib/chr*_phastCons.wig
    mkdir -p /gbdb/hg17/phastCons/wib
    rm -f /gbdb/hg17/phastCons/wib/chr*phastCons.wib
    ln -s /cluster/data/hg17/bed/multiz.2004-07-13/cons/run.elements/wib/*.wib /gbdb/hg17/phastCons/wib
    chmod 775 . wib /gbdb/hg17/phastCons /gbdb/hg17/phastCons/wib
    chmod 664 wib/*.wib

    # move postprobs over and clean up bluearc 
    rsync -av /cluster/bluearc/hg17/phastCons/POSTPROBS .
    # (people sometimes want the raw scores)
    rm -r /cluster/bluearc/hg17/phastCons/ELEMENTS /cluster/bluearc/hg17/phastCons/POSTPROBS /cluster/bluearc/hg17/phastCons/wib

    # set up full alignment/conservation track ("multiz8way")
    # load multiz maf tables 
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz.2004-07-13
    set mafDir = /gbdb/hg17/multiz8way/maf
    set table = multiz8way
    mkdir -p $mafDir/$table
    ln -s `pwd`/maf/*.maf $mafDir/$table
    cd maf
    hgLoadMaf hg17 -warn multiz8way -pathPrefix=$mafDir/$table/maf

    # someone dropped from hgwdev
    # reload (2007-03-19 kate)
    nice hgLoadMaf hg17 -warn multiz8way -pathPrefix=/gbdb/hg17/multiz8wayFixed
    cat /gbdb/hg17/multiz8wayFixed/*.maf | \
        nice hgLoadMafSummary hg17 -minSize=30000 -mergeGap=1500 -maxSize=200000  \
            multiz8waySummary stdin

    # load blastz maf tables
    # TODO: change mafWiggle to use db names instead of species names
    # in speciesOrder 
    # link files into /gbdb table dir
    ln -s /cluster/data/hg17/bed/blastz.panTro1/mafNet $mafDir/chimp_netBlastz
    ln -s /cluster/data/hg17/bed/blastz.mm5/mafNet $mafDir/mouse_netBlastz
    ln -s /cluster/data/hg17/bed/blastz.rn3/mafNet $mafDir/rat_netBlastz
    ln -s /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/mafNet $mafDir/dog_netBlastz
    ln -s /cluster/data/hg17/bed/blastz.galGal2/mafNet $mafDir/chicken_netBlastz
    ln -s /cluster/data/hg17/bed/blastz.fr1/mafNet $mafDir/fugu_netBlastz
    ln -s /cluster/data/hg17/bed/blastz.danRer1.swap/mafNet $mafDir/zebrafish_netBlastz

    # remove empty file, disliked by hgLoadMaf
    # NOTE: these shouldn't be empty -- next time, make sure previous
    # alignments are copied over to output maf (multiz won't if there's
    # an empty input file).
    rm chicken/chr18_random.maf
    rm fugu/chr6_hla_hap1.maf

    # load tables
    foreach s (chimp mouse rat dog chicken fugu zebrafish)
        set table = ${s}_netBlastz
        echo "$s $mafDir/$table"
        ~kate/bin/i386/hgLoadMaf hg17 -warn ${s}_netBlastz -pathPrefix=$mafDir/$table
    end

    # trackDb entry:
# track multiz8way
# shortLabel Conservation
# longLabel Chimp/Mouse/Rat/Dog/Chicken/Fugu/Zebrafish Multiz Alignments & Conservation
# group compGeno
# priority 149
# visibility pack
#color 0, 10, 100
# type wigMaf 0.0 1.0
# maxHeightPixels 100:40:11
# wiggle phastCons
# yLineOnOff Off
# autoScale Off
# pairwise netBlastz
# speciesOrder chimp mouse rat dog chicken fugu zebrafish


# PHASTCONS SCORES DOWNLOADABLES (REDONE 6/15/05 angie)
    # Initially done 10/11/04, but using scores from run.cons -- which 
    # had been replaced by scores in run.elements, where I did not think 
    # to look for scores.  :(  !
    ssh eieio
    mkdir /cluster/data/hg17/zip/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1
    cd /cluster/data/hg17/bed/multiz8way/cons/run.elements/POSTPROBS
    foreach chr (`awk '{print $1;}' /cluster/data/hg17/chrom.sizes`)
      echo $chr
      nice zcat `ls -1 $chr.*.pp.gz | sort -t\. -k2,2n` \
      | nice gzip -c \
      > /cluster/data/hg17/zip/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1/$chr.gz
    end
    ssh hgwdev
    mkdir /usr/local/apache/htdocs/goldenPath/hg17/phastCons
    # Doh!  /cluster/data/hg17/zip/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1 is 11G now -- 
    # too much to dump on hgwdev's / which is at 94%.  So don't do this:
    #mv /cluster/data/hg17/zip/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1 .
    # make symbolic links instead:
    mkdir /usr/local/apache/htdocs/goldenPath/hg17/phastCons/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1
    cd /usr/local/apache/htdocs/goldenPath/hg17/phastCons/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1
    ln -s /cluster/data/hg17/zip/mzPt1Mm5Rn3Cf1Gg2Fr1Dr1/* .
    md5sum *.gz > md5sum.txt
    # make a README.txt.


# PHYLOFIT AND TREE-DOCTOR FOR 8-WAY: ESTIMATE PHYLOGENETIC TREE (acs)
# (This was originally done for phastCons but is not necessary with
# the new version.  However, it may be useful for other purposes, so
# I'm leaving it in as a separate entry.)

    # first estimate a model for the mammals
    ssh eieio
    cd /cluster/bluearc/hg17/multiz.2004-07-13/panTro1mm5rn3canFam1

    # collect sufficient stats (takes maybe an hour)
    for file in *.maf ; do echo $file ; msa_view -i MAF $file -o SS --order hg17,panTro1,rn3,mm5,canFam1 > `basename $file .maf`.ss ; done
    ls *.ss | grep -v chr6_hla_hap2 > files
    msa_view '*files' --aggregate hg17,panTro1,rn3,mm5,canFam1 -i SS -o SS > all.ss
    # BTW, this can now be done in one step using something like:
    # msa_view --aggregate hg17,panTro1,rn3,mm5,canFam1 -i MAF -o SS *.maf > all.ss
    # (modify to exclude certain files if necessary)

    # estimate model, with rate variation (takes about a minute)
    phyloFit all.ss --nrates 10 --tree "(((hg17,panTro1),(rn3,mm5)),canFam1)" --alpha 4.4 --EM --log log -i SS --out-root hprmc-rev-dg
    # (Actually, --nrates 4 should be more than adequate for most purposes)

    cat hprmc-rev-dg.mod
#ALPHABET: A C G T
#ORDER: 0
#SUBST_MOD: REV
#NRATECATS: 10
#ALPHA: 4.658942
#TRAINING_LNL: -6889216721.159384
#BACKGROUND: 0.294633 0.205082 0.205189 0.295097
#RATE_MAT:
#  -0.865237    0.159990    0.554805    0.150442
#   0.229851   -1.194646    0.168269    0.796526
#   0.796651    0.168182   -1.194919    0.230086
#   0.150205    0.553556    0.159985   -0.863747
#TREE: (((1:0.006523,2:0.007997):0.103779,(3:0.104867,4:0.078911):0.265676):0.112364,5:0.112364);

    # now extrapolate to fish and chicken using tree_doctor and the CFTR 25 tree 
    # (replace numbers with names in hprmc-rev-dg.mod; this won't be necessary in the future)
    tree_doctor --rename "1->hg17;2->panTro1;3->rn3;4->mm5;5->canFam1" hprmc-rev-dg.mod > hprmc-rev-dg.names.mod
    # (obtain 8-way subtree from cftr25_hybrid.nh; also map names as necessary to match above)
    tree_doctor /cluster/data/nisc/targets/cftr/phyloHMMcons25/cftr25_hybrid.nh --prune-all-but hg16,chimp,mm3,rn3,dog,chicken,fr1,zfish --rename "hg16->hg17;mm3->mm5;chimp->panTro1;dog->canFam1;chicken->galGal2;zfish->danRer1" > cftr8way.nh
    # now merge (see tree_doctor help page for explanation)
    tree_doctor hprmc-rev-dg.names.mod --merge cftr8way.nh > hg17panTro1rn3mm5canFam1galGal2fr1danRer1.mod

    cat hg17panTro1rn3mm5canFam1galGal2fr1danRer1.mod
#ALPHABET: A C G T
#ORDER: 0
#SUBST_MOD: REV
#NRATECATS: 10
#ALPHA: 4.658942
#BACKGROUND: 0.294633 0.205082 0.205189 0.295097
#RATE_MAT:
#  -0.865237    0.159990    0.554805    0.150442
#   0.229851   -1.194646    0.168269    0.796526
#   0.796651    0.168182   -1.194919    0.230086
#   0.150205    0.553556    0.159985   -0.863747
#TREE: (((((hg17:0.006523,panTro1:0.007997):0.103779,(rn3:0.104867,mm5:0.078911):0.265676):0.019461,canFam1:0.205267):0.377150,galGal2:0.511134):0.536627,(danRer1:0.905323,fr1:0.922995):0.536627);


# CONSERVED NON-CODING (CNS) TRACK (acs 08/29/04)
# (depends on phastConsElements)

    cd /cluster/data/hg17/bed/multiz.2004-07-13/cons/run.elements
    featureBits hg17 -bed=possibleCoding.bed -or twinscan:exon xenoMrna mrna intronEst 
    # (add SGP, exoniphy, possib. others if available)
    # now filter out all phastCons elements that overlap
    overlapSelect -nonOverlapping possibleCoding.bed all.bed cns.bed
    hgLoadBed hg17 cns cns.bed

# track cns
# shortLabel CNS
# longLabel Conserved Non-Coding (Cons Elements Minus Predicted Coding)
# priority 109.11
# group compGeno
# visibility hide
# type bed 5 .

    
# PRODUCING GENSCAN PREDICTIONS (DONE - 2004-07-08 - Hiram)
#	Needed to download a new binary for this run.  Our Linux systems
#  XXX - I thought a new binary was needed.  Turned out it was already
#  here in our hg3rdParty CVS project.  All of this discussed here can
#  be simply fetched from cvs:  cvs co hg3rdParty/genscanlinux
#	have been updated since the last time, the old binary would not
#	run.  Go to: http://genes.mit.edu/GENSCAN.html
#	and then to: http://genes.mit.edu/license.html
#	Fill in the license agreement and you can then pick up the
#	README and the Linux version: genscanlinux.tar.uue.tgz
#	To uudecode that file, go to one of the Solaris home machines
#	and use the uudecode command:
#	uudecode genscanlinux.tar.uue.tgz
#	That produces the file: genscanlinux.tar
#	Which contains the files:
# drwxr-xr-x chris/burgelab    0 2003-02-17 11:48:44 ./
# -rw-r--r-- chris/burgelab 219056 2000-09-07 12:39:26 ./Arabidopsis.smat
# -rw-r--r-- chris/burgelab   6622 2000-09-07 12:39:26 ./HUMRASH
# -rw-r--r-- chris/burgelab    849 2000-09-07 12:39:26 ./HUMRASH.sample
# -rw-r--r-- chris/burgelab 219050 2000-09-07 12:39:26 ./HumanIso.smat
# -rw-r--r-- chris/burgelab 155735 2000-09-07 12:39:26 ./Maize.smat
# -rw-r--r-- chris/burgelab  24465 2000-09-07 12:39:26 ./README
# -rw-r--r-- chris/burgelab   6344 2000-09-07 12:39:27 ./HUMRASH.ps
# -rwxr-xr-x chris/burgelab 126365 2003-02-17 11:48:44 ./genscan
#
#	I placed these currently in: /cluster/home/hiram/GENSCAN/
#	I'll check with Angie where it should properly live ...
#  XXX - it already lives in 'cvs co hg3rdParty/genscanlinux'
#  These instructions should simple check it out right here in
#  bed/genscan and make the gsub command refer to these copies.

    ssh hgwdev
    mkdir /cluster/data/hg17/bed/genscan
    cd /cluster/data/hg17/bed/genscan
    cvs co hg3rdParty/genscanlinux

    ssh eieio
    cd /cluster/data/hg17/bed/genscan
    # Make 3 subdirectories for genscan to put their output files in
    mkdir gtf pep subopt
    # Generate a list file, genome.list, of all the contigs
    # *that do not have pure Ns* (due to heterochromatin, unsequencable 
    # stuff) which would cause genscan to run forever.
    rm -f genome.list
    bash
    for f in `cat /cluster/data/hg17/contig.lst`
    do
      egrep '[ACGT]' /cluster/data/hg17/$f.masked > /dev/null
	if [ $? = 0 ]; then
	    echo /cluster/data/hg17/$f.masked >> genome.list
	fi
    done
    # exit your bash shell if you are [t]csh ...
    #	This egrep matched all the contigs in hg17.  I guess none of
    #	them are complete Ns* at this point.

    # Log into kki (not kk !).  kki is the driver node for the small
    # cluster (kkr2u00 -kkr8u00. Genscan has problem running on the
    # big cluster, due to limitation of memory and swap space on each
    # processing node).
    ssh kki
    cd /cluster/data/hg17/bed/genscan
    # Create template file, gsub, for gensub2.  For example (3-line file):
    cat << '_EOF_' > gsub
#LOOP
/cluster/bin/x86_64/gsBig {check in line+ $(path1)} {check out line gtf/$(root1).gtf} -trans={check out line pep/$(root1).pep} -subopt={check out line subopt/$(root1).bed} -exe=hg3rdParty/genscanlinux/genscan -par=hg3rdParty/genscanlinux/HumanIso.smat -tmp=/tmp -window=2400000
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy
    gensub2 genome.list single gsub jobList
    para create jobList
    para try
    para check
    para push ... etc ...
# Completed: 379 of 380 jobs
# Crashed: 1 jobs
# CPU time in finished jobs:      79998s    1333.30m    22.22h    0.93d  0.003 y
# IO & Wait Time:                  2989s      49.82m     0.83h    0.03d  0.000 y
# Average job time:                 219s       3.65m     0.06h    0.00d
# Longest job:                     2999s      49.98m     0.83h    0.03d
# Submission to last job:          8324s     138.73m     2.31h    0.10d

    #	Running the single failed job on kolossus with a smaller window:
/cluster/bin/x86_64/gsBig /cluster/data/hg17/5/NT_006576/NT_006576.fa.masked \
        gtf/NT_006576.fa.gtf -trans=pep/NT_006576.fa.pep \
        -subopt=subopt/NT_006576.fa.bed -exe=hg3rdParty/genscanlinux/genscan \
        -par=hg3rdParty/genscanlinux/HumanIso.smat -tmp=/tmp -window=2000000

    # If there were out-of-memory problems (run "para problems"), then 
    # re-run those jobs by hand but change the -window arg from 2400000
    # something lower.  In build33, this was 22/NT_011519
    #  In build34 there were NO failures !

    # Convert these to chromosome level files as so:     
    ssh eieio
    cd /cluster/data/hg17/bed/genscan
    $HOME/bin/i386/liftUp genscan.gtf ../../jkStuff/liftAll.lft warn gtf/N*.gtf
    $HOME/bin/i386/liftUp genscanSubopt.bed ../../jkStuff/liftAll.lft \
	warn subopt/N*.bed
    cat pep/*.pep > genscan.pep

    # Load into the database as so:
    ssh hgwdev
    cd /cluster/data/hg17/bed/genscan
    ldHgGene hg17 genscan genscan.gtf
    #	35 minute job
    #	Read 42807 transcripts in 325994 lines in 1 files
    #	  42807 groups 46 seqs 1 sources 1 feature types

    hgPepPred hg17 generic genscanPep genscan.pep
    #	Processing genscan.pep
    hgLoadBed hg17 genscanSubopt genscanSubopt.bed
    #	Reading genscanSubopt.bed
    #	Loaded 517157 elements of size 6
    #	Sorted
    #	Creating table definition for 
    #	Saving bed.tab
    #	Loading hg17

    #	featureBits hg17 genscan
    #	55323340 bases of 2866216770 (1.930%) in intersection

    #	featureBits hg16 genscan
    #	55333689 bases of 2865248791 (1.931%) in intersection

    #	featureBits hg17 genscanSubopt
    #	55986178 bases of 2866216770 (1.953%) in intersection
    #	featureBits hg16 genscanSubopt
    #	56082952 bases of 2865248791 (1.957%) in intersection

    #	Should be zero intersection with rmsk
    #	featureBits -chrom=chr1 hg17 genscan rmsk
    #	794 bases of 222827847 (0.000%) in intersection


# EXTRACT LINEAGE-SPECIFIC REPEATS FOR DOG (DONE 8/1/05 angie)
    # Originally done 7/1/04 for canFam1 -- redone 8/1/05 for canFam2.
    ssh kolossus
    cd /san/sanvol1/scratch/hg17/rmsk
    # Run Arian's DateRepsinRMoutput.pl to add extra columns telling 
    # whether repeats in -query are also expected in -comp species.  
    # Even though we already have the human-mouse linSpecReps,
    # extractLinSpecReps requires two columns of DateRepsinRMoutput.pl
    # additions.  So add mouse, then ignore it.  
    # Dog in extra column 1, Mouse in extra column 2
    foreach outfl ( *.out )
        echo "$outfl"
        /cluster/bluearc/RepeatMasker/DateRepeats \
          ${outfl} -query human -comp dog -comp mouse
    end
    # Now extract dog (extra column 1), ignore mouse.
    cd ..
    mkdir linSpecRep.notInDog
    foreach f (rmsk/*.out_canis-familiaris_mus-musculus)
        set base = $f:t:r:r
        echo $base.out.spec
        /cluster/bin/scripts/extractLinSpecReps 1 $f > \
                        linSpecRep.notInDog/$base.out.spec
    end
    # Clean up.
    rm rmsk/*.out_canis*
    rsync -av /san/sanvol1/scratch/hg17/linSpecRep.notInDog \
      /cluster/bluearc/scratch/hg/gs.18/build35/
    # Ask cluster-admin for an rsync.


# BLASTZ DOG (CANFAM1) (DONE 7/8/04 angie)
    ssh kk
    # space is awful tight on store4 -- use store7.  
    mkdir -p /cluster/store7/hg17/bed/blastz.canFam1.2004-07-08
    ln -s /cluster/store7/hg17/bed/blastz.canFam1.2004-07-08 \
      /cluster/data/hg17/bed/
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08
    # Use default (Human-Mouse) settings for starters.
    cat << '_EOF_' > DEF
# human vs. dog
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin

ALIGN=blastz-run
BLASTZ=blastz

# Default
BLASTZ_H=2000
BLASTZ_ABRIDGE_REPEATS=1

# TARGET: Human
SEQ1_DIR=/scratch/hg/gs.18/build35/bothMaskedNibs
SEQ1_RMSK=
SEQ1_FLAG=
SEQ1_SMSK=/scratch/hg/gs.18/build35/linSpecRep.notInDog
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Dog
SEQ2_DIR=/scratch/hg/canFam1/nib
SEQ2_RMSK=
SEQ2_FLAG=
SEQ2_SMSK=/scratch/hg/canFam1/linSpecRep.notInHuman
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/blastz.canFam1.2004-07-08

DEF=$BASE/DEF
RAW=$BASE/raw
CDBDIR=$BASE
SEQ1_LEN=$BASE/S1.len
SEQ2_LEN=$BASE/S2.len
'_EOF_'
    # << this line keeps emacs coloring happy

    # first cluster run: raw blastz alignments
    ssh kk
    bash # if a csh/tcsh user
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08
    source DEF
    mkdir $RAW run.0
    /cluster/home/angie/hummus/make-joblist $DEF > $BASE/run.0/j
    sh ./xdir.sh
    cd run.0
    sed -e 's@^blastz-run@/cluster/bin/penn/blastz-run@' j > jobList
    para create jobList
    para try, check, push, check, ....
    # Moving the human chr19 jobs up to the top of the jobList probably 
    # would have shaved 4 hours off the total time!  It was almost done 
    # after 6 hours, except for a few chr19 stragglers.
#Completed: 93775 of 93775 jobs
#Average job time:                 202s       3.37m     0.06h    0.00d
#Longest job:                    17806s     296.77m     4.95h    0.21d
#Submission to last job:         35523s     592.05m     9.87h    0.41d

    # second cluster run: lift raw alignments -> lav dir
    ssh kki
    bash # if a csh/tcsh user
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08
    source DEF
    mkdir run.1 lav
    /cluster/bin/scripts/blastz-make-out2lav $DEF $BASE > $BASE/run.1/jobList
    cd run.1
    wc -l jobList
    para create jobList
    para try, check, push, etc ...
#Completed: 341 of 341 jobs
#Average job time:                  36s       0.61m     0.01h    0.00d
#Longest job:                      302s       5.03m     0.08h    0.00d
#Submission to last job:          1143s      19.05m     0.32h    0.01d

    # third run: lav -> axt
    # (if non-default BLASTZ_Q is used in the future, put axtRescore in 
    # the pipe after lavToAxt)
    ssh kki
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08
    mkdir axtChrom pslChrom run.2
    cd run.2
    cat << '_EOF_' > do.csh
#!/bin/csh -ef
cd $1
set chr = $1:t
cat `ls -1 *.lav | sort -g` \
| $HOME/bin/x86_64/lavToAxt stdin \
    /iscratch/i/gs.18/build35/bothMaskedNibs /iscratch/i/canFam1/nib stdout \
| $HOME/bin/x86_64/axtSort stdin ../../axtChrom/$chr.axt 
$HOME/bin/x86_64/axtToPsl ../../axtChrom/$chr.axt ../../S1.len ../../S2.len \
  ../../pslChrom/$chr.psl
'_EOF_'
    # << this line keeps emacs coloring happy
    chmod a+x do.csh
    cp /dev/null jobList
    foreach d (../lav/chr*)
      echo "do.csh $d" >> jobList
    end
    para create jobList
    para try, check, push, check
#Completed: 46 of 46 jobs
#Average job time:                 300s       5.00m     0.08h    0.00d
#Longest job:                     1669s      27.82m     0.46h    0.02d
#Submission to last job:          1689s      28.15m     0.47h    0.02d


# CHAIN DOG BLASTZ (DONE 7/9/04 angie)
    # Run axtChain on little cluster
    ssh kki
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08
    mkdir -p axtChain/run1
    cd axtChain/run1
    mkdir out chain
    ls -1S /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChrom/*.axt \
      > input.lst
    cat << '_EOF_' > gsub
#LOOP
doChain {check in line+ $(path1)} {check out line+ chain/$(root1).chain} {check out exists out/$(root1).out}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    cat << '_EOF_' > doChain
#!/bin/csh
axtChain $1 \
    /iscratch/i/gs.18/build35/bothMaskedNibs \
    /iscratch/i/canFam1/nib $2 > $3
'_EOF_'
    # << this line makes emacs coloring happy
    chmod a+x doChain
    gensub2 input.lst single gsub jobList
    para create jobList
    para try, check, push, check...
#Completed: 46 of 46 jobs
#Average job time:                 266s       4.43m     0.07h    0.00d
#Longest job:                     3578s      59.63m     0.99h    0.04d
#Submission to last job:          3578s      59.63m     0.99h    0.04d
    # now on the cluster server, sort chains
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
    chainMergeSort run1/chain/*.chain > all.chain
    chainSplit chain all.chain
    rm run1/chain/*.chain

    # take a look at score distr's
    foreach f (chain/*.chain)
      grep chain $f | awk '{print $2;}' | sort -nr > /tmp/score.$f:t:r
      echo $f:t:r
      textHistogram -binSize=10000 /tmp/score.$f:t:r
      echo ""
    end

    # Lots of chaff with scores in the 3000's.  Many very-high-scoring 
    # chains.  So filter the chain down somewhat...
    mv all.chain all.chain.unfiltered
    chainFilter -minScore=5000 all.chain.unfiltered > all.chain
    rm chain/*
    chainSplit chain all.chain
    gzip all.chain.unfiltered

    # Load chains into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain/chain
    foreach i (*.chain)
        set c = $i:r
        hgLoadChain hg17 ${c}_chainCanFam1 $i
    end
    # Coverage is significantly higher than mouse:
    featureBits hg17 -chrom=chr1 chainCanFam1Link
#123999291 bases of 222827847 (55.648%) in intersection
# before filtering: 124750124 bases of 222827847 (55.985%) in intersection
    featureBits hg17 -chrom=chr1 chainMm5Link
#83773012 bases of 222827847 (37.595%) in intersection


# NET DOG BLASTZ (DONE 7/9/04 angie)
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
    chainPreNet all.chain ../S1.len ../S2.len stdout \
    | chainNet stdin -minSpace=1 ../S1.len ../S2.len stdout /dev/null \
    | netSyntenic stdin noClass.net

    # Add classification info using db tables:
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
    netClass noClass.net hg17 canFam1 dog.net

    # Make a 'syntenic' subset:
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
    rm noClass.net
    # Make a 'syntenic' subset of these with
    netFilter -syn dog.net > dogSyn.net

    # Load the nets into database 
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
    netFilter -minGap=10 dog.net |  hgLoadNet hg17 netCanFam1 stdin
    netFilter -minGap=10 dogSyn.net | hgLoadNet hg17 syntenyNetCanFam1 stdin
    # Add entries for chainCanFam1, netCanFam1 to human/hg17 trackDb


# MAKE VSCANFAM1 DOWNLOADABLES (DONE 9/17/04 kate)
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
    ln -s all.chain dog.chain
    mkdir gz
    cd gz
    gzip -c ../dog.chain > dog.chain.gz
    gzip -c ../dog.net > dog.net.gz
    gzip ../dogSyn.net > dogSyn.net.gz
    # Angie's notes...
    # Mike Zody asked for raw blastz in chain format, so figure out some 
    # way to translate axt or psl to chain and put it out there.  
    # Actually, it's probably just hg16-canFam1 that he wants for now -- ?
    # Ask when we get to this point.

    cd ../axtNet
    time gzip *.axt

    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17
    mkdir -p vsCanFam1
    cd vsCanFam1
    mv /cluster/data/hg17/bed/blastz.canFam1/axtChain/gz/*.gz .
    md5sum *.gz > md5sum.txt
    mkdir -p axtNet
    cd axtNet
    cp /cluster/data/hg17/bed/blastz.canFam1/axtNet/*.axt.gz .
    md5sum *.gz > md5sum.txt
    # Copy over & edit README.txt w/pointers to chain, net formats.

    # REDO downloads of axtNet's to fix overlaps (2005-09-13 kate)
    # Finally, replace bad chr5 files (2006-01-05 kate)
    ssh kkstore02
    cd /cluster/data/hg17/bed/blastz.canFam1/axtNet
    nice gzip *.axt
    md5sum *.axt.gz > md5sum.txt

    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsCanFam1
    mv axtNet axtNet.old
    ln -s /cluster/data/hg17/bed/blastz.canFam1/axtNet .


# GENERATE CANFAM1 MAF FOR MULTIZ FROM NET (DONE 7/9/04 angie)
# Redo net axt's and maf's to fix overlaps (use 8/5 netToAxt)
#       (2005-08-16 kate)
# and replace bad chr5 files (2006-01-05 kate)
    ssh kkstore02
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08/axtChain
    netSplit dog.net net
    cd /cluster/data/hg17/bed/blastz.canFam1.2004-07-08
    mkdir axtNet mafNet
cat > makeMaf.csh << 'EOF'
    foreach f (axtChain/net/*)
      set chr = $f:t:r
      echo $chr
      netToAxt $f axtChain/chain/$chr.chain /cluster/data/hg17/nib \
        /cluster/data/canFam1/nib stdout \
      | axtSort stdin axtNet/$chr.axt
      axtToMaf axtNet/$chr.axt \
            /cluster/data/hg17/chrom.sizes /cluster/data/canFam1/chrom.sizes \
            mafNet/$chr.maf -tPrefix=hg17. -qPrefix=canFam1.
    end
'EOF'
    csh makeMaf.csh >&! makeMaf.log &
    mkdir -p /cluster/bluearc/hg17/mafNet
    cp -rp mafNet /cluster/bluearc/hg17/mafNet/canFam1


# BLASTZ MM5 (DONE - 2004-06-22 - Hiram)
    ssh kk
    mkdir -p /cluster/data/hg17/bed/blastz.mm5.2004-07-01
    cd /cluster/data/hg17/bed
    ln -s  blastz.mm5.2004-07-01 blastz.mm5
    cd blastz.mm5

    cat << '_EOF_' > DEF
# human vs. mouse
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/home/angie/schwartzbin:/cluster/home/kent/bin/i386

ALIGN=blastz-run
BLASTZ=blastz
BLASTZ_H=2000
BLASTZ_ABRIDGE_REPEATS=1

# TARGET
# Human
SEQ1_DIR=/iscratch/i/gs.18/build35/bothMaskedNibs
# not used
SEQ1_RMSK=
# not used
SEQ1_FLAG=
SEQ1_SMSK=/iscratch/i/gs.18/build35/linSpecRep.notInRat
# notInRat OK as it is identical to notInMouse
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY
# Mouse
SEQ2_DIR=/scratch/mus/mm5/softNib
# RMSK not currently used
SEQ2_RMSK=/scratch/mus/mm5/rmsk
# FLAG not currently used
SEQ2_FLAG=-rodent
SEQ2_SMSK=/scratch/mus/mm5/linSpecRep.notInHuman
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=30000000
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/blastz.mm5

DEF=$BASE/DEF
RAW=$BASE/raw
CDBDIR=$BASE
SEQ1_LEN=$BASE/S1.len
SEQ2_LEN=$BASE/S2.len
'_EOF_'
    # << this line keeps emacs coloring happy


    # prepare first cluster run
    ssh kk
    cd /cluster/data/hg17/bed/blastz.mm5
    /cluster/data/hg17/jkStuff/BlastZ_run0.sh
    cd run.0
    para try, check, push, check, ....
# Completed: 44330 of 44330 jobs
# CPU time in finished jobs:   16250628s  270843.80m  4514.06h  188.09d  0.515 y
# IO & Wait Time:                387936s    6465.60m   107.76h    4.49d  0.012 y
# Average job time:                 375s       6.26m     0.10h    0.00d
# Longest job:                     4417s      73.62m     1.23h    0.05d
# Submission to last job:         43754s     729.23m    12.15h    0.51d

    #	the file server to its knees.  Run this on the small cluster.
    ssh kki
    cd /cluster/data/hg17/bed/blastz.mm5
    /cluster/data/hg17/jkStuff/BlastZ_run1.sh
    cd run.1
    para try, check, push, etc ...
# Completed: 341 of 341 jobs
# CPU time in finished jobs:       2189s      36.48m     0.61h    0.03d  0.000 y
# IO & Wait Time:                  7714s     128.57m     2.14h    0.09d  0.000 y
# Average job time:                  29s       0.48m     0.01h    0.00d
# Longest job:                      165s       2.75m     0.05h    0.00d
# Submission to last job:           830s      13.83m     0.23h    0.01d


    #	Third cluster run to convert lav's to axt's
    #	Does not work on kki since /scratch on the iservers is not the
    #	same as /scratch on the other clusters.
    ssh kk
    cd /cluster/data/hg17/bed/blastz.mm5
    /cluster/data/hg17/jkStuff/BlastZ_run2.sh
    cd run.2
    para try, check, push, etc ...
# Completed: 45 of 46 jobs
# Crashed: 1 jobs
# CPU time in finished jobs:       1638s      27.30m     0.46h    0.02d  0.000 y
# IO & Wait Time:                 12068s     201.13m     3.35h    0.14d  0.000 y
# Average job time:                 305s       5.08m     0.08h    0.00d
# Longest job:                     1124s      18.73m     0.31h    0.01d
# Submission to last job:          2519s      41.98m     0.70h    0.03d
    #	chr19 takes too long, the axtSort becomes too large and the poor
    #	node ends up swapping forever.  When you are down to that last
    #	job running, stop it and go to kolossus.
    #	Adjusting the location of the nib directories, and fixing the
    #	MACHTYPE on the commands in the blastz script:
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.mm5
    sed -e "s/i386/x86_64/g" /cluster/bin/scripts/blastz-chromlav2axt > \
	x86_64-chromlav2axt
    chmod +x x86_64-chromlav2axt
    time ./x86_64-chromlav2axt \
	/cluster/data/hg17/bed/blastz.mm5/lav/chr19 \
	/cluster/data/hg17/bed/blastz.mm5/axtChrom/chr19.axt \
	/cluster/bluearc/scratch/hg/gs.18/build35/bothMaskedNibs \
	/cluster/bluearc/scratch/mus/mm5/softNib
    #	real    7m41.719s
    #	user    2m2.850s
    #	sys     0m23.070s

    # translate sorted axt files into psl
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm5
    mkdir -p pslChrom
    set tbl = "blastzMm5"
    foreach f (axtChrom/chr*.axt)
      set c=$f:t:r
      echo "Processing chr $c"
      /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
    end
    #	This takes more than an hour.  You can shorten this by changing
    #	that command to a simple echo, put the results into a file,
    #	split the file into four parts and run the four files as shell
    #	scripts on eieio to have four processes running at the same
    #	time.  Load on eieio gets up to about 20 which is reasonable.

    # Load database tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm5/pslChrom
    bash		#	for tcsh users
    for F in chr*_blastzMm5.psl
    do
	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${F}
	echo "${F} done"
    done
    # this is a 40 minute job
    # exit bash if you are tcsh

    # featureBits on blastzMm3 or 4 will not work on hgwdev, runs out of
    # memory.  But if you reset your ~/.hg.conf to use the read-only
    #	user and contact the hgwdev host, then use the x86_64 featureBits
    # featureBits hg16 blastzMm5
    # 1056761609 bases of 2865248791 (36.882%) in intersection
    # featureBits hg17 blastzMm5
    #	1052077141 bases of 2866216770 (36.706%) in intersection
    # featureBits hg17 blastzMm4
    #  1056201417 bases of 2866216770 (36.850%) in intersection

# CHAIN MM5 BLASTZ (DONE - 2004-07-02 - Hiram)

# The axtChain is best run on the small kluster, or the kk9 kluster
    ssh kki
    mkdir -p /cluster/data/hg17/bed/blastz.mm5/axtChain/run1
    cd /cluster/data/hg17/bed/blastz.mm5/axtChain/run1
    mkdir out chain

    ls -1S /cluster/data/hg17/bed/blastz.mm5/axtChrom/*.axt > input.lst
    cat << '_EOF_' > gsub
#LOOP
doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} out/$(root1).out
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

#  May need -minScore=5000 for all chroms if chr19 won't finish on kolossus

    cat << '_EOF_' > doChain
#!/bin/csh
axtChain $1 /iscratch/i/gs.18/build35/bothMaskedNibs \
	/iscratch/i/mus/mm5/softNib $2 > $3
'_EOF_'
    # << this line makes emacs coloring happy
    chmod a+x doChain

    # 46 jobs
    gensub2 input.lst single gsub jobList
    para create jobList
    para try
    para push # ... etc ...
# Completed: 46 of 46 jobs
# CPU time in finished jobs:       4856s      80.94m     1.35h    0.06d  0.000 y
# IO & Wait Time:                 20083s     334.71m     5.58h    0.23d  0.001 y
# Average job time:                 542s       9.04m     0.15h    0.01d
# Longest job:                     2929s      48.82m     0.81h    0.03d
# Submission to last job:          2929s      48.82m     0.81h    0.03d


    # now on the file server, sort chains
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm5/axtChain
    time chainMergeSort run1/chain/*.chain > all.chain
    #	real    8m42.853s
    #	user    5m59.100s
    #	sys     0m40.320s

    time chainSplit chain all.chain
    #	real    10m52.224s
    #	user    5m52.360s
    #	sys     0m34.870s

    # optionally: rm run1/chain/*.chain

    # Load chains into database
    # next machine
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm5/axtChain/chain
    bash	#	for tcsh users
    for i in *.chain
    do
        c=${i/.chain/}
        hgLoadChain hg17 ${c}_chainMm5 $i
        echo done $c
    done
    # exit bash if you are tcsh
    #	This is a 50 minute job

    #	featureBits hg17 chainMm5
    #	2834490112 bases of 2866216770 (98.893%) in intersection
    #	featureBits hg17 chainMm4
    #	2829135227 bases of 2866216770 (98.706%) in intersection
    #	featureBits hg16 chainMm4
    #	2828363353 bases of 2865248791 (98.713%) in intersection


# NET MM5 (DONE - 2004-07-02 - Hiram)
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm5/axtChain
    mkdir preNet
    cd chain
    bash	#	for tcsh users
    for i in *.chain
    do
      echo preNetting $i
      /cluster/bin/i386/chainPreNet $i /cluster/data/hg17/chrom.sizes \
                        /cluster/data/mm5/chrom.sizes ../preNet/$i
    done
    # exit bash if you are tcsh
    #	15 minute job

    cd ..
    mkdir n1
    cd preNet
    bash	#	for tcsh users
    for i in *.chain
    do
      n=${i/.chain/}.net
      echo primary netting $i $n
      /cluster/bin/i386/chainNet $i -minSpace=1 /cluster/data/hg17/chrom.sizes \
	/cluster/data/mm5/chrom.sizes ../n1/$n /dev/null
    done
    # exit bash if you are tcsh
    #	9 minute job

    cd ..
    cat n1/*.net | /cluster/bin/i386/netSyntenic stdin hNoClass.net
    #	memory usage 2546110464, utime 16327 s/100, stime 3546

    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm5/axtChain
    time netClass hNoClass.net hg17 mm5 mouse.net \
	-tNewR=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInMouse \
	-qNewR=/cluster/bluearc/scratch/mus/mm5/linSpecRep.notInHuman
    #	real    16m38.098s
    #	user    11m38.490s
    #	sys     1m48.470s

    # If things look good do
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm5/axtChain
    rm -r n1 hNoClass.net
    # Make a 'syntenic' subset of these with
    time netFilter -syn mouse.net > mouseSyn.net
    #	real    12m3.701s
    #	user    8m44.180s
    #	sys     1m1.610s

    # Load the nets into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm5/axtChain
    netFilter -minGap=10 mouse.net |  hgLoadNet hg17 netMm5 stdin
    netFilter -minGap=10 mouseSyn.net | hgLoadNet hg17 syntenyNetMm5 stdin

    # check results
    #	featureBits hg17 netMm5
    #	2830625630 bases of 2866216770 (98.758%) in intersection

    #	featureBits hg17 netMm4
    #	2824272033 bases of 2866216770 (98.537%) in intersection
    # featureBits hg16 netMm5
    #	2823565051 bases of 2865248791 (98.545%) in intersection

    # featureBits hg17 syntenyNetMm5
    #	2799194300 bases of 2866216770 (97.662%) in intersection
    # featureBits hg17 syntenyNetMm4
    #	2785830955 bases of 2866216770 (97.195%) in intersection
    # featureBits hg16 syntenyNetMm5
    #	2786960572 bases of 2865248791 (97.268%) in intersection

    # Add entries for net and chain to mouse/hg17 trackDb

    # make net
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm5/axtChain
    mkdir mouseNet
    time netSplit mouse.net mouseNet
    #	real    11m45.243s
    #	user    8m48.490s
    #	sys     1m13.490s

    #	extract axt's from net, and convert to maf's
    # NOTE: Redo the net axt's and maf's using 8/05 netToAxt
    # in order to remove overlaps (2005-08-16 kate)
    ssh kkstore02
    cd /cluster/data/hg17/bed/blastz.mm5/axtChain
    mkdir ../axtNet ../mafNet
cat > makeMaf.csh << '_EOF_'
#!/bin/csh -ef
    foreach f (mouseNet/chr*.net)
        set c = $f:t:r
        echo "netToAxt: $c.net -> $c.axt"
        rm -f ../axtNet/$c.axt
        netToAxt mouseNet/$c.net chain/$c.chain \
	    /cluster/data/hg17/nib /cluster/data/mm5/nib stdout | \
	    axtSort stdin ../axtNet/$c.axt
        axtToMaf ../axtNet/$c.axt \
            /cluster/data/hg17/chrom.sizes /cluster/data/mm5/chrom.sizes \
            ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=mm5.
	echo "Complete: $c.net -> axtNet/$c.axt -> mafNet/$c.maf"
    end
'_EOF_'
# << for emacs
    csh makeMaf.csh >&! makeMaf.log &
    tail -100f makeMaf.log
    mkdir -p /cluster/bluearc/hg17/mafNet
    cp -rp ../mafNet /cluster/bluearc/hg17/mafNet/mm5

    ssh hgwdev
    mkdir /cluster/data/hg17/bed/blastz.mm5/axtBest
    cd /cluster/data/hg17/bed/blastz.mm5/axtBest
    ln -s ../axtNet/chr*.axt .

    # copy net axt's to download area
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm5/axtNet
    mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/axtNet
    cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/axtNet
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/axtNet
    gzip *.axt
    # add README.txt file to dir (use previous assembly's copy as template)
    #	32 minute gzip
    

    #  Convert those axt files to psl
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm5
    mkdir pslBest
    foreach a (axtBest/chr*.axt)
	set c=$a:t:r
	echo -n "processing $c.axt -> ${c}_blastzBestMm5.psl ..."
    /cluster/bin/i386/axtToPsl axtBest/${c}.axt \
	S1.len S2.len pslBest/${c}_blastzBestMm5.psl
	echo "Done"
    end

    # Load tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm5/pslBest
    for I in chr*BestMm5.psl
    do
	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
	echo "done ${I}"
    done

     # check results
    # featureBits hg17 blastzBestMm5
    #	1013348528 bases of 2866216770 (35.355%) in intersection
    # featureBits hg17 blastzBestMm4
    #	1017319919 bases of 2866216770 (35.493%) in intersection
    # featureBits hg16 blastzBestMm5
    #	996722004 bases of 2865248791 (34.787%) in intersection

    # Make /gbdb links and add them to the axtInfo table:
     mkdir -p /gbdb/hg17/axtBest/Mm5
     cd /gbdb/hg17/axtBest/Mm5
     ln -s /cluster/data/hg17/bed/blastz.mm5/axtNet/chr*.axt .
     cd /cluster/data/hg17/bed/blastz.mm5/axtNet
     rm -f axtInfoInserts.sql
     foreach f (/gbdb/hg17/axtBest/Mm5/chr*.axt)
       set chr=$f:t:r
       echo "INSERT INTO axtInfo (species, alignment, chrom, fileName) \
                VALUES ('mm5','Blastz Best in Genome','$chr','$f');" \
         >>! axtInfoInserts.sql
     end
    hgsql hg17 < ~/kent/src/hg/lib/axtInfo.sql
    #	table axtInfo may already exist, ignore create error.
    hgsql hg17 < axtInfoInserts.sql

    # REDO: replace downloadable axtNet's to remove overlaps (2005-09-12 kate)
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsMm5
    mv axtNet axtNet.old
    mkdir axtNet
    cd axtNet
    cp /cluster/data/hg17/bed/blastz.mm5/axtNet/*.axt .
    nice gzip *.axt
    md5sum *.axt.gz > md5sum.txt


# HG17 TO MM5 LIFTOVER CHAIN (DONE 1/6/05 Andy)
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.mm5/axtChain
    mkdir over
    for file in chain/*.chain; do
       chrom=`basename $file .chain`
       netChainSubset mouseNet/$chrom.net chain/$chrom.chain over/$chrom.over
       cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToMm5.chain
    done
    rm -rf over/
    ssh hgwdev
    mkdir /usr/local/apache/htdocs/goldenPath/hg17/liftOver
    cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver
    cp /cluster/data/hg17/bed/liftOver/hg17ToMm5.chain .
    gzip hg17ToMm5.chain
    ln -s /cluster/data/hg17/bed/liftOver/hg17ToMm5.chain /gbdb/hg17/liftOver/hg17ToMm5.over.chain
    hgAddLiftOverChain -multiple hg17 mm5

# HG17 TO CANFAM1 LIFTOVER CHAIN (DONE 1/7/05 Andy)
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.canFam1/axtChain
    mkdir over
    for file in chain/*.chain; do
       chrom=`basename $file .chain`
       netChainSubset net/$chrom.net chain/$chrom.chain over/$chrom.over
       cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToCanFam1.chain
    done
    rm -rf over/
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver
    cp /cluster/data/hg17/bed/liftOver/hg17ToCanFam1.chain .
    gzip hg17ToCanFam1.chain
    ln -s /cluster/data/hg17/bed/liftOver/hg17ToCanFam1.chain /gbdb/hg17/liftOver/hg17ToCanFam1.over.chain
    hgAddLiftOverChain -multiple hg17 canFam1

# HG17 TO PANTRO1 LIFTOVER CHAIN (DONE 1/20/05 Andy)
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.panTro1/axtChain
    mkdir over
    for file in chain/*.chain; do
       chrom=`basename $file .chain`
       netChainSubset net/$chrom.net chain/$chrom.chain over/$chrom.over
       cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToPanTro1.chain
    done
    rm -rf over/
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver
    cp /cluster/data/hg17/bed/liftOver/hg17ToPanTro1.chain .
    gzip hg17ToPanTro1.chain
    ln -s /cluster/data/hg17/bed/liftOver/hg17ToPanTro1.chain /gbdb/hg17/liftOver/hg17ToPanTro1.over.chain
    hgAddLiftOverChain -multiple hg17 panTro1   

# HG17 TO RN3 LIFTOVER CHAIN (DONE 3/1/05 Andy)
    #ssh kolossus
    #cd /cluster/data/hg17/bed/blastz.rn3/axtChain
    #mkdir over
    #for file in chain/*.chain; do
    #   chrom=`basename $file .chain`
    #   netChainSubset ratNet/$chrom.net chain/$chrom.chain over/$chrom.over
    #   cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToRn3.chain
    #done
    #rm -rf over/
    # Oh fancy that, there's already a hg17ToRn3.over.chain in the /cluster/data/hg17/bed/liftOver
    # directory generated by Angie.  
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver 
    cp /cluster/data/hg17/bed/liftOver/hg17ToRn3.over.chain .
    gzip hg17ToRn3.over.chain
    ln -s /cluster/data/hg17/bed/liftOver/hg17ToRn3.over.chain /gbdb/hg17/liftOver/hg17ToRn3.over.chain
    hgAddLiftOverChain -multiple hg17 rn3   

# HG17 TO GALGAL2 LIFTOVER CHAIN (DONE 3/1/05 Andy)
    # OK there's already a /cluster/data/hg17/bed/liftOver/hg17ToGalGal2.over.chain file generated
    # by Angie.
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver 
    cp /cluster/data/hg17/bed/liftOver/hg17ToGalGal2.over.chain .
    gzip hg17ToGalGal2.over.chain
    ln -s /cluster/data/hg17/bed/liftOver/hg17ToGalGal2.over.chain /gbdb/hg17/liftOver/hg17ToGalGal2.over.chain
    hgAddLiftOverChain -multiple hg17 galGal2       

# HG17 TO MONDOM1 LIFTOVER CHAIN (DONE 3/1/05 Andy)
    ssh kksilo
    cd /cluster/data/monDom1/bed/zb.hg17/axtChain
    netSplit human.net.gz net
    ssh kolossus
    cd /cluster/data/monDom1/bed/zb.hg17/axtChain
    mkdir over
    for file in chain/*.chain.gz; do
       chrom=`basename $file .chain.gz`
       netChainSubset net/$chrom.net chain/$chrom.chain.gz over/$chrom.over
       cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToMonDom1.chain
    done
    rm -rf over/ net/
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver 
    cp /cluster/data/hg17/bed/liftOver/hg17ToMonDom1.chain .
    gzip hg17ToMonDom1.chain
    ln -s /cluster/data/hg17/bed/liftOver/hg17ToMonDom1.chain /gbdb/hg17/liftOver/hg17ToMonDom1.over.chain
    hgAddLiftOverChain -multiple hg17 monDom1 

# HG17 TO DANRER2 LIFTOVER CHAIN (DONE 3/2/05 Andy)
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
    chainSplit chain all.chain.gz
    netSplit zfishdanRer2.net.gz net
    mkdir over
    # FAILED STEPS:
    #for file in chain/*.chain; do
    #   chrom=`basename $file .chain`
    #   netChainSubset net/$chrom.net chain/$chrom.chain over/$chrom.over
    #   cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToDanRer2.chain
    #done
    # Error:
#read 28019 of 28019 chains in chain/chr1.chain
#Processing chr1
#netChainSubset: netChainSubset.c:55: writeChainPart: Assertion `subChain != ((void *)0)' failed.
    # OK instead of using the ones in the chain/ subdir, I'm using the ones in
    # the chainAR/ subdir.  These chain files had an additional step in the process of making
    # them: Rachel used the chainAntiRepeat program.  
    for file in chain/*.chain; do
       chrom=`basename $file .chain`
       if [ $chrom = "chr1" ]; then
           netChainSubset net/$chrom.net chainAR/$chrom.chain over/$chrom.over
       else
           netChainSubset net/$chrom.net chainAR/$chrom.chain.gz over/$chrom.over          
       fi
       cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToDanRer2.chain
    done  
    rm -rf over/
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver 
    cp /cluster/data/hg17/bed/liftOver/hg17ToDanRer2.chain .
    gzip hg17ToDanRer2.chain
    ln -s /cluster/data/hg17/bed/liftOver/hg17ToDanRer2.chain /gbdb/hg17/liftOver/hg17ToDanRer2.over.chain
    hgAddLiftOverChain -multiple hg17 danRer2 
    

# HG17 TO TETNIG1 LIFTOVER CHAIN (DONE 3/1/05 Andy)
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
    mkdir over
    for file in chain/*.chain; do
       chrom=`basename $file .chain`
       netChainSubset tetraodonNet/$chrom.net chain/$chrom.chain over/$chrom.over
       cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToTetNig1.chain
    done
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver 
    cp /cluster/data/hg17/bed/liftOver/hg17ToTetNig1.chain .
    gzip hg17ToTetNig1.chain
    ln -s /cluster/data/hg17/bed/liftOver/hg17ToTetNig1.chain /gbdb/hg17/liftOver/hg17ToTetNig1.over.chain
    hgAddLiftOverChain -multiple hg17 tetNig1      

# HG17 TO BOSTAU1 LIFTOVER CHAIN (DONE Mar. 18, 2004, Heather)

    ssh kolossus
    cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
    mkdir over
    for file in chain/*.chain; do
       chrom=`basename $file .chain`
       netChainSubset net/$chrom.net chain/$chrom.chain over/$chrom.over
       cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToBosTau1.chain
    done
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver 
    cp /cluster/data/hg17/bed/liftOver/hg17ToBosTau1.chain .
    gzip hg17ToBosTau1.chain
    ln -s /cluster/data/hg17/bed/liftOver/hg17ToBosTau1.chain /gbdb/hg17/liftOver/hg17ToBosTau1.over.chain
    hgAddLiftOverChain -multiple hg17 bosTau1      

# HG17 TO XENTRO1 LIFTOVER CHAIN (DONE 7/5/05 Andy)
    ssh kolossus
    cd /cluster/data/xenTro1/bed/zb.hg17/axtChain
    mkdir chain net over
    chainSplit chain all.chain
    netSplit human.net net
    for file in chain/*.chain; do
       chrom=`basename $file .chain`
       netChainSubset net/$chrom.net chain/$chrom.chain over/$chrom.over
       cat over/$chrom.over >> /cluster/data/hg17/bed/liftOver/hg17ToXenTro1.chain
    done
    rm -rf over/ chain/ net/
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver
    cp /cluster/data/hg17/bed/liftOver/hg17ToXenTro1.chain .
    gzip hg17ToXenTro1.chain
    ln -s /cluster/data/hg17/bed/liftOver/hg17ToXenTro1.chain /gbdb/hg17/liftOver/hg17ToXenTro1.over.chain
    hgAddLiftOverChain -multiple hg17 xenTro1   

# ADD CHAIN AND NET TO VSMM5 AND VSRN3 DOWNLOAD AREAS (DONE 8/5/04 angie)
    ssh hgwdev
    cp -p /cluster/data/hg17/bed/blastz.mm5/axtChain/all.chain.gz \
      /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/mouse.chain.gz
    cp -p /cluster/data/hg17/bed/blastz.mm5/axtChain/mouse.net.gz \
      /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/mouse.net.gz
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsMm5
    md5sum *.gz */*.gz > md5sum.txt
    # Update the README.txt
    cp -p /cluster/data/hg17/bed/blastz.rn3/axtChain/all.chain.gz \
      /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/rat.chain.gz
    cp -p /cluster/data/hg17/bed/blastz.rn3/axtChain/rat.net.gz \
      /usr/local/apache/htdocs/goldenPath/hg17/vsRn3/rat.net.gz
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsRn3
    md5sum *.gz */*.gz > md5sum.txt
    # Update the README.txt

# ADD CHAIN AND NET TO VSHG17 DOWNLOAD AREAS (DONE Sept. 8th, 2004, heather)
    ssh hgwdev
    cp -p /cluster/data/mm5/bed/blastz.hg17/axtChain/all.chain.gz \
      /usr/local/apache/htdocs/goldenPath/mm5/vsHg17/human.chain.gz
    cp -p /cluster/data/mm5/bed/blastz.hg17/axtChain/human.net.gz \
      /usr/local/apache/htdocs/goldenPath/mm5/vsHg17/human.net.gz
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsRn3
    md5sum *.gz */*.gz > md5sum.txt
    # Update the README.txt


# SWAP BLASTZ ZEBRAFISH-HUMAN (danRer1-hg17) to HUMAN-ZEBRAFISH (hg17-danRer1)
# USE RESCORED ALIGNMENTS (see makeDanRer1.doc)
# (DONE, 2004-06-22, hartera)
# CONVERT AXTs TO PSL AND LOAD INTO DATABASE (DONE, 2004-07-08, hartera)
    ssh kolossus
    mkdir /cluster/data/hg17/bed/blastz.danRer1.swap
    cd /cluster/data/hg17/bed/blastz.danRer1.swap
    # use rescored axtChrom from blastzHg17 on danRer1
    set aliDir = /cluster/data/danRer1/bed/blastz.hg17
    cp $aliDir/S1.len S2.len
    cp $aliDir/S2.len S1.len
    mkdir unsorted axtChrom
    cat $aliDir/axtChrom/chr*.axt \
    | axtSwap stdin $aliDir/S1.len $aliDir/S2.len stdout \
    | axtSplitByTarget stdin unsorted
    # Sort the shuffled .axt files.
    foreach f (unsorted/*.axt)
      echo sorting $f:t:r
      axtSort $f axtChrom/$f:t
    end
    du -sh $aliDir/axtChrom unsorted axtChrom
# 19G     /cluster/data/danRer1/bed/blastz.hg17/axtChrom
# 19G     unsorted
    rm -r unsorted
   # translate sorted axt files into psl
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.danRer1.swap
    mkdir -p pslChrom
    set tbl = "blastzDanRer1"
    foreach f (axtChrom/chr*.axt)
      set c=$f:t:r
      echo "Processing chr $c"
      /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
    end
    # Load database tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.danRer1.swap/pslChrom
    foreach f (./*.psl)
      /cluster/bin/i386/hgLoadPsl hg17 $f
      echo "$f Done"
    end

# CHAIN ZEBRAFISH (danRer1) BLASTZ (DONE, 2004-06-23, hartera)
    # Run axtChain on little cluster
    ssh kki
    cd /cluster/data/hg17/bed/blastz.danRer1.swap
    mkdir -p axtChain/run1
    cd axtChain/run1
    mkdir out chain
    ls -1S /cluster/data/hg17/bed/blastz.danRer1.swap/axtChrom/*.axt \
                    > input.lst
    cat << '_EOF_' > gsub
#LOOP
doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
#ENDLOOP
'_EOF_'

    # << this line makes emacs coloring happy

# Reuse gap penalties from hg16 vs chicken run.
    cat << '_EOF_' > ../../chickenHumanTuned.gap
tablesize^V     11
smallSize^V     111
position^V      1^V     2^V     3^V     11^V    111^V   2111^V  12111^V 32111^V 72111^V 152111^V        252111
qGap^V  325^V   360^V   400^V   450^V   600^V   1100^V  3600^V  7600^V  15600^V 31600^V 56600
tGap^V  325^V   360^V   400^V   450^V   600^V   1100^V  3600^V  7600^V  15600^V 31600^V 56600
bothGap^V       625^V   660^V   700^V   750^V   900^V   1400^V  4000^V  8000^V  16000^V 32000^V 57000
'_EOF_'
    # << this line makes emacs coloring happy

cat << '_EOF_' > doChain
#!/bin/csh
axtFilter $1 \
| axtChain -scoreScheme=/cluster/data/blastz/HoxD55.q \
                      -linearGap=../../chickenHumanTuned.gap \
                      -minScore=5000 stdin \
    /iscratch/i/gs.18/build35/bothMaskedNibs \
    /iscratch/i/danRer1/nib $2 > $3
'_EOF_'
    chmod a+x doChain
    gensub2 input.lst single gsub jobList
    para create jobList
    para try, check, push, check...
# para time
# Completed: 45 of 46 jobs
# Crashed: 1 jobs
# CPU time in finished jobs:       3559s      59.32m     0.99h    0.04d  0.000 y
# IO & Wait Time:                   934s      15.56m     0.26h    0.01d  0.000 y
# Average job time:                 100s       1.66m     0.03h    0.00d
# Longest job:                      502s       8.37m     0.14h    0.01d
# Submission to last job:          2969s      49.48m     0.82h    0.03d
# chr19.axt crashed - out of memory so try again on kolossus

    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain/run1
    # need to use nibs on bluearc as iscratch not accessible to kolossus
cat << '_EOF_' > doChain2
#!/bin/csh
axtFilter $1 \
| axtChain -scoreScheme=/cluster/data/blastz/HoxD55.q \
                      -linearGap=../../chickenHumanTuned.gap \
                      -minScore=5000 stdin \
    /cluster/bluearc/hg17/bothMaskedNibs \
    /cluster/bluearc/danRer1/nib $2 >& $3
'_EOF_'
    chmod +x doChain2

    doChain2 \
         /cluster/data/hg17/bed/blastz.danRer1.swap/axtChrom/chr19.axt \
         chain/chr19.chain out/chr19.out
    # now on the cluster server, sort chains
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
    chainMergeSort run1/chain/*.chain > all.chain
    chainSplit chain all.chain

    # Load chains into database
    # next machine
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain/chain
    foreach i (*.chain)
        set c = $i:r
        hgLoadChain hg17 ${c}_chainDanRer1 $i
        echo done $c
    end
# tried minScore = 1000 and minScore = 10000 for axtChain
# minScore = 5000 was best for reducing low scoring chains but not reducing 
# overlap with refGene CDS too much

# NET ZEBRAFISH (danRer1) BLASTZ (DONE, 2004-06-24, hartera)
# REMAKE NET WITHOUT ANCIENT REPEATS (DONE, 2004-07-07, hartera)
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
    mkdir preNet
    cd chain
    foreach i (*.chain)
       echo preNetting $i
       /cluster/bin/i386/chainPreNet $i ../../S1.len ../../S2.len \
                                     ../preNet/$i
    end

    cd ..
    mkdir n1
    cd preNet
    foreach i (*.chain)
      set n = $i:r.net
      echo primary netting $i
      /cluster/bin/i386/chainNet $i -minSpace=1 ../../S1.len ../../S2.len \
                                 ../n1/$n /dev/null
    end

    cd ..
    mkdir n1
    cd preNet
    foreach i (*.chain)
      set n = $i:r.net
      echo primary netting $i
      /cluster/bin/i386/chainNet $i -minSpace=1 ../../S1.len ../../S2.len \
                                 ../n1/$n /dev/null
    end

    cd ..
    cat n1/*.net | /cluster/bin/i386/netSyntenic stdin noClass.net
    # memory usage 149086208, utime 868 s/100, stime 173                                                                            
    # Add classification info using db tables:
    cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
    # netClass looks for ancient repeats in one of the databases
    # hg17 has this table - hand-curated by Arian
    # this is only for human rodent comparisons so use -noAr option
    mkdir -p /cluster/bluearc/danRer1/linSpecRep.notInHuman
    mkdir -p /cluster/bluearc/hg17/linSpecRep.notInZebrafish
    cp /iscratch/i/gs.18/build35/linSpecRep.notInZebrafish/* \
       /cluster/bluearc/hg17/linSpecRep.notInZebrafish
    cp /iscratch/i/danRer1/linSpecRep.notInHuman/* \
       /cluster/bluearc/danRer1/linSpecRep.notInHuman
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
    # add -noAr option
    # mkdir old
    # mv zebrafish.net ./old/zebrafish.net.old
    time netClass noClass.net hg17 danRer1 zebrafish.net \
        -tNewR=/cluster/bluearc/hg17/linSpecRep.notInZebrafish \
        -qNewR=/cluster/bluearc/danRer1/linSpecRep.notInHuman -noAr
    # 83.410u 43.650s 3:09.94 66.8%   0+0k 0+0io 198pf+0w
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
    netFilter -minGap=10 zebrafish.net | hgLoadNet hg17 netDanRer1 stdin

# EXTRACT AXT'S AND MAF'S FROM ZEBRAFISH (danRer1) NET 
# (DONE, 2004-06-24, hartera) used net where hg17 ancient Repeat table used
# sorted axts and remade mafs as multiz needs axts to be sorted
# (DONE, 2004-06-25, kate)
# Redone to fix overlaps using 8/05 axtToNet (2005-08-16 kate)
# Replace bad chr5 axtNet and mafNet (2006-01-05 kate) 
    ssh eieio
    # create axts
    cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
    netSplit zebrafish.net zebrafishNet
    mkdir -p ../axtNet ../mafNet
cat > makeMaf.csh << 'EOF'
    foreach f (zebrafishNet/chr*.net)
        set c = $f:t:r
        echo $c
        netToAxt zebrafishNet/$c.net chain/$c.chain \
        /cluster/data/hg17/nib /cluster/data/danRer1/nib stdout | \
               axtSort stdin ../axtNet/$c.axt
        axtToMaf ../axtNet/$c.axt \
            /cluster/data/hg17/chrom.sizes /cluster/data/danRer1/chrom.sizes \
            ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=danRer1.
    end
'EOF'
    csh makeMaf.csh >&! makeMaf.log &
    tail -100f makeMaf.log
    mkdir -p /cluster/bluearc/hg17/mafNet
    cp -rp ../mafNet /cluster/bluearc/hg17/mafNet/danRer1


# BLASTZ ZEBRAFISH (danRer1) CLEAN UP (DONE, 2004-07-19, hartera)
# FURTHER CLEANUP (hartera, 2006-09-01, hartera)
     ssh eieio
     cd /cluster/data/hg17/bed/blastz.danRer1.swap
     nice rm axtChain/run1/chain/* &
     nice rm -fr axtChain/n1 axtChain/hNoClass.net &
     nice gzip axtChrom/* pslChrom/* axtChain/all.chain axtChain/*.net &
     # further cleanup (2006-09-01, hartera)
     ssh kkstore02
     cd /cluster/data/hg17/bed/blastz.danRer1.swap
     rm -r axtNet.old axtNet.unsorted mafNet
     cd axtChain
     rm hist*
     # remove chains and nets directories. These can be reconstructed with
     # all.chain.gz and zebrafish.net.gz
     rm -r old chain zebrafishNet preNet
     rm noClass.net.gz
     cd ..
     rm pslChrom/psl.tab.gz

# ZEBRAFISH DANRER1 DOWNLOADS (WORKING 2004-09-17 kate)
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtNet
    gzip *.axt

    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17
    mkdir -p vsDanRer1
    cd vsDanRer1
    cp /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain/all.chain.gz zebrafish.chain.gz
    cp /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain/zebrafish.net.gz .
    md5sum *.gz > md5sum.txt
    mkdir -p axtNet
    cd axtNet
    cp /cluster/data/hg17/bed/blastz.danRer1.swap/axtNet/*.axt.gz .
    md5sum *.gz > md5sum.txt
    # Copy and edit README.txt
      
 
# MAKING MOUSE SYNTENY (DONE - 2004-07-03 - Hiram)

ssh hgwdev
mkdir /cluster/data/hg17/bed/syntenyMm5
cd /cluster/data/hg17/bed/syntenyMm5

# Copy all the needed scripts from /cluster/data/hg16/bed/syntenyMm3
cp -p /cluster/data/hg17/bed/syntenyRn3/*.pl .

./syntenicBest.pl -db=hg17 -table=blastzBestMm5
./smooth.pl
./joinsmallgaps.pl
./fillgap.pl -db=hg17 -table=blastzBestMm5
./synteny2bed.pl
    #	The five commands above
    #	real    209m28.161s
    #	user    0m21.040s
    #	sys     0m4.100s

#	Used to load this in syntenyMm5, but that type is misleading to
#	the table browser and fails the checkTableCoords check.
#	Better to use this ensRatMusHom type:
#	Need a new name here for the Mm5 to not conflict with Rn3
sed -e 's/ensPhusionBlast/ensRatMm5Hom/g' \
      $HOME/kent/src/hg/lib/ensPhusionBlast.sql \
      > ensRatMm5Hom.sql
hgLoadBed hg17 ensRatMm5Hom ucsc100k.bed -sqlTable=ensRatMm5Hom.sql
    #	featureBits hg17 ensRatMm5Hom
    #	2649530748 bases of 2866216770 (92.440%) in intersection
    #	featureBits hg17 ensRatMm4Hom
    #	2549307611 bases of 2866216770 (88.943%) in intersection
    #	featureBits hg16 syntenyMm5
    #	2560252977 bases of 2865248791 (89.355%) in intersection

# MAKING MOUSE AXTTIGHT FROM AXTBEST (DONE - 2004-07-02 - Hiram)
    # After creating axtBest alignments above, use subsetAxt to get axtTight:
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm5/axtNet
    mkdir -p ../axtTight
    bash	#	for tcsh users
    for I in *.axt
    do
      echo $I
      subsetAxt  $I ../axtTight/$I \
	~kent/src/hg/mouseStuff/subsetAxt/coding.mat 3400
    done
    # exit bash if you are tcsh
    #	An 8 minute job

    # translate to psl
    cd ../axtTight
    mkdir ../pslTight
    bash	#	for tcsh users
    for I in *.axt
    do
      C=${I/.axt/}
      axtToPsl $I ../S1.len ../S2.len ../pslTight/${C}_blastzTightMm5.psl
      echo "Done: $I -> ${C}_blastzTightMm5.psl"
    done
    # exit bash if you are tcsh

    # Load tables into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm5/pslTight
    for I in chr*TightMm5.psl
    do
	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
	echo "done ${I}"
    done

    #	Compare results with previous assembly:
    #	featureBits hg17 blastzTightMm5
    #	165862935 bases of 2866216770 (5.787%) in intersection
    #	featureBits hg17 blastzTightMm4
    #	166569246 bases of 2866216770 (5.811%) in intersection
    #	featureBits hg16 blastzTightMm5
    #	162641577 bases of 2865248791 (5.676%) in intersection

    # copy  axt's to download area
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.mm5/axtTight
    mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/axtTight
    cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/axtTight
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsMm5/axtTight
    gzip *.axt
    # add README.txt file to dir (use previous assembly's copy as template)
    #	4 minute gzip

# BLASTZ MM5 CLEAN UP (DONE 2004-07-02 - Hiram)
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.mm5
    nice rm -rf raw &
    nice rm -fr axtChain/n1 axtChain/hNoClass.net &
    nice rm axtChain/run1/chain/* &
    nice gzip axtChrom/* pslChrom/* lav/*/* axtChain/all.chain axtChain/*.net &

##############################################################################
#  MAKING BLASTZ SELF (DONE - 2004-07-14 - Hiram)

    # The procedure for lineage spec business with self is to simply
    # use the actual repeat masker output for this human assembly as
    # the lineage specific repeats for itself.  Thus, merely make
    # symlinks to the repeat masker out files and name them as expected
    # for blastz.  In this case they are called notInHuman but they
    # really mean InHuman.  Yes, it is confusing, but that's just the
    # nature of the game in this case.

    ssh eieio
    mkdir /cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInHuman
    cd /cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInHuman
    foreach f (../rmsk/*.fa.out)
	set base = $f:t:r:r
	echo $base.out.spec
	ln -s $f $base.out.spec
    end
    #	Same thing done on iscratch
    #	Not worried about pushing this scratch yet, it will get done
    #	sometime later.  Using the actual /cluster/bluearc/scratch/
    #	location below.

    ssh kk
    mkdir /cluster/data/hg17/bed/blastzSelf.2004-07-01
    cd /cluster/data/hg17/bed
    ln -s blastzSelf.2004-07-01 blastzSelf
    cd blastzSelf

    cat << '_EOF_' > DEF
# human vs. human
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/home/angie/schwartzbin:/cluster/home/kent/bin/i386

ALIGN=blastz-run
BLASTZ=blastz
BLASTZ_H=2000
BLASTZ_ABRIDGE_REPEATS=1

# TARGET
# Human
SEQ1_DIR=/iscratch/i/gs.18/build35/bothMaskedNibs
# not used
SEQ1_RMSK=
# not used
SEQ1_FLAG=
SEQ1_SMSK=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInHuman
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY
# Human
SEQ2_DIR=/iscratch/i/gs.18/build35/bothMaskedNibs
# not currently used
SEQ2_RMSK=
# not currently used
SEQ2_FLAG=
SEQ2_SMSK=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInHuman
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=10000000
SEQ2_LAP=10000

BASE=/cluster/data/hg17/bed/blastzSelf

DEF=$BASE/DEF
RAW=$BASE/raw
CDBDIR=$BASE
SEQ1_LEN=$BASE/S1.len
SEQ2_LEN=$BASE/S2.len
'_EOF_'
    # << this line makes emacs coloring happy

    # prepare first cluster run
    ssh kk
    cd /cluster/data/hg17/bed/blastzSelf
    /cluster/data/hg17/jkStuff/BlastZ_run0.sh
    cd run.0
    para try, check, push, check, ....
    #	you need a -maxPush=200000 on this one, it is more than 100000
    #	jobs the default push limit.  Also be aware of maxQueue limits
    #	on the KK, may need something more than the default of 200000 if
    #	the KK is busy.
XXX - running 2004-07-01 11:26


##############################################################################
# LIFTOVER (DROP) CHAINS TO HG16 (IN PROGRESS 2005-01-03 kate)

   # swap hg16->hg17 chains 

 # LIFTOVER (DROP) CHAINS TO HG16 (IN PROGRESS 2004-07-07 kate)

    # run alignment
    # NOTE: split hg16 to /iscratch/i is doc'ed in makeHg16.doc
    ssh kk
    cd /cluster/data/hg17
    makeLoChain-align hg17 /scratch/hg/gs.18/build35/bothMaskedNibs \
                    hg16 /iscratch/i/gs.17/build34/liftOver/split
    # Created parasol job in bed/blat.hg16.2004-07-07/run
    # 1150 jobs
    cd bed/blat.hg16.2004-07-07/run
    para try
    para check
    para push

    # GOT HERE

    # lift results (use bash)
    cd /cluster/data/hg17/bed/blat.hg16
    for file in /cluster/data/hg16/nib/*.nib; do
       chrom=`basename $file .nib`
       liftUp -pslQ psl/$chrom.psl /cluster/bluearc/hg/gs.17/build34/liftOver/lift/$chrom.lft warn raw/chr*_${chrom}.psl
    done
    # There were some errors from not finding .lft files for the chr_random ones.
    ssh kk9
    cd ../liftOver
    ln -s blat.hg16 blat.hg16.2005-01-22
    makeLoChain-chain hg17 /cluster/data/hg17/nib hg16 /cluster/data/hg16/nib 2>chain.error.log >chain.log
    ssh eieio
    makeLoChain-net hg17 hg16
    ssh hgwdev
    makeLoChain-load hg17 hg16

# DROPUNDER CHAIN TO HG15 (DONE 2005-07-21 Andy)
    # Split things up
    ssh eieio
    cd /cluster/bluearc
    mkdir -p hg15/liftOver/split
    cd hg15/liftOver/split/
    mkdir ../lift
    for c in `cut -f1 /cluster/data/hg15/chrom.sizes`; do
       echo $c
       num=${c%_random}
       num=${num#chr}
       faSplit -lift=../lift/${c}.lft size /cluster/data/hg15/${num}/${c}.fa -oneFile 3000 ${c}
    done  

    # Move files to santest
    ssh hgwdev
    cd /santest/scratch
    mkdir hg15
    cd hg15/
    cp -r /cluster/bluearc/hg15/liftOver .

    # run alignment
    ssh kk
    cd /cluster/data/hg17
    makeLoChain-align hg17 /scratch/hg/gs.18/build35/bothMaskedNibs \
                    hg15 /santest/scratch/hg15/liftOver/split
    # Created parasol job in bed/blat.hg16.2004-07-07/run
    # 2024 jobs written to batch
    # *** IGNORE the batch created by the script.
    ln -s bed/blat.hg15.2005-07-21 bed/blat.hg15
    cd bed/blat.hg15/
    mv run run.kk
    mkdir run.kk9 run.kki
    cd run.kk/
    sed 's/\.fa\./\./g' spec > tmp; mv tmp spec
    grep Un_random spec > ../run.kki/spec
    grep -v Un_random spec > newspec
    mv newspec spec
    egrep "chr(1|19|X)(\.|_)" spec | grep -v random > ../run.kk9/spec
    grep -Fv -f ../run.kk9/spec spec > newspec
    mv newspec spec
    wc -l spec ../run.kk9/spec ../run.kki/spec
#   1831 spec
#    147 ../run.kk9/spec
#     46 ../run.kki/spec
#   2024 total
    # Checks out
    
    # Run the thing on all 3 clusters.
    para create spec
    para push
#Completed: 1831 of 1831 jobs
#CPU time in finished jobs:    8556066s  142601.10m  2376.69h   99.03d  0.271 y
#IO & Wait Time:                 60428s    1007.13m    16.79h    0.70d  0.002 y
#Average job time:                4706s      78.43m     1.31h    0.05d
#Longest running job:                0s       0.00m     0.00h    0.00d
#Longest finished job:           46724s     778.73m    12.98h    0.54d
#Submission to last job:         46725s     778.75m    12.98h    0.54d
    ssh kk9
    cd /cluster/data/hg17/bed/blat.hg15/run.kk9
    para create spec
    para push
#Completed: 147 of 147 jobs
#CPU time in finished jobs:    1698424s   28307.07m   471.78h   19.66d  0.054 y
#IO & Wait Time:                   874s      14.56m     0.24h    0.01d  0.000 y
#Average job time:               11560s     192.66m     3.21h    0.13d
#Longest running job:                0s       0.00m     0.00h    0.00d
#Longest finished job:           31413s     523.55m     8.73h    0.36d
#Submission to last job:         31413s     523.55m     8.73h    0.36d
    ssh kki
    cd /cluster/data/hg17/bed/blat.hg15/run.kki
    para create spec
    para push
    # OK I don't have para time stuff for this one, but it was the shortest 
    # by far.

    # lift results
    cd /cluster/data/hg17/bed/blat.hg15/lift.run
    for chrom in `cut -f1 /cluster/data/hg15/chrom.sizes`; do
       liftUp -pslQ /cluster/bluearc/hg15/liftOver/psl/${chrom}.psl /cluster/bluearc/hg15/liftOver/lift/${chrom}.lft warn raw/chr*_${chrom}.psl
    done

    # Chain
    # There's been some problems with store5.  
    ssh kk9
    cd /cluster/store12/store5/gs.18/build35/bed/blat.hg15.2005-07-21
    mkdir chainRun
    mkdir -p /panasas/store/hg15/chainRaw
    ln -s /panasas/store/hg15/chainRaw chainRaw
    cd chainRun/
    ls -1S ../psl/*.psl > in.lst
    cat > chain.sh << "_EOF_"
#!/bin/bash
tmp=/scratch/`basename $4`
axtChain -psl $1 $2 $3 $tmp
cp $tmp $4
rm $tmp  
_EOF_
    chmod +x chain.sh
    cat > gsub << "_EOF_"
#LOOP
./chain.sh $(path1) /scratch/hg/gs.18/build35/bothMaskedNibs /scratch/hg/gs.16/build33/chromTrfMixedNib {check out line+ ../chainRaw/$(root1).chain}
#ENDLOOP
_EOF_
    # <<
    cd chainRun/
    gensub2 in.lst single gsub spec
    para create spec
    para push
#Completed: 44 of 44 jobs
#CPU time in finished jobs:       7448s     124.13m     2.07h    0.09d  0.000 y
#IO & Wait Time:                  9591s     159.85m     2.66h    0.11d  0.000 y
#Average job time:                 387s       6.45m     0.11h    0.00d
#Longest running job:                0s       0.00m     0.00h    0.00d
#Longest finished job:            1906s      31.77m     0.53h    0.02d
#Submission to last job:          1906s      31.77m     0.53h    0.02d
    ssh kolossus
    cd /panasas/store/hg15/chainRaw    
    chainMergeSort *.chain | chainSplit /scratch/andy/chain stdin
    cd /scratch/andy
    mkdir net over
    cd chain/
    for chain in *; do
       c=${chain%.chain}
       echo $c
       chainNet $chain /cluster/store12/store5/gs.18/build35/chrom.sizes \
          /cluster/store12/store5/gs.16/build33/chrom.sizes \
          ../net/${c}.net /dev/null
       netChainSubset ../net/${c}.net $chain ../over/${c}.over
    done
    cd ../over/
    cat * >> ../hg17ToHg15.over.chain
    cd ../
    cp -r hg17* over/ /cluster/store12/store5/gs.18/build35/bed/blat.hg15.2005-07-21/
    cd ../
    rm -rf andy/
    rm -rf /panasas/store/hg15    
    cd /cluster/bluearc/hg15/liftOver/psl
    for psl in *; do
       gzip $psl
    done
    cd ../

    
# Completed: 116281 of 116281 jobs
# CPU time in finished jobs:   21807388s  363456.46m  6057.61h  252.40d  0.692 y
# IO & Wait Time:               2319383s   38656.39m   644.27h   26.84d  0.074 y
# Average job time:                 207s       3.46m     0.06h    0.00d
# Longest job:                    22063s     367.72m     6.13h    0.26d
# Submission to last job:         83402s    1390.03m    23.17h    0.97d


    #	Second cluster run to convert the .out's to .lav's
    #	You do NOT want to run this on the big cluster.  It brings
    #	the file server to its knees.  Run this on the small cluster.
    ssh kki
    cd /cluster/data/hg17/bed/blastzSelf
    /cluster/data/hg17/jkStuff/BlastZ_run1.sh
    cd run.1
    para try, check, push, etc ...

# Completed: 341 of 341 jobs
# CPU time in finished jobs:       6344s     105.73m     1.76h    0.07d  0.000 y
# IO & Wait Time:                  5413s      90.22m     1.50h    0.06d  0.000 y
# Average job time:                  34s       0.57m     0.01h    0.00d
# Longest job:                      505s       8.42m     0.14h    0.01d
# Submission to last job:          4521s      75.35m     1.26h    0.05d

    #	Third cluster run to convert lav's to axt's

    #	These self alignments do not work well as the usual third cluster job.
    #	Instead, a specialized job here that includes a DropSelf
    #	operation, and in individual lav pieces to avoid out of memory
    #	problems during axtSort
    ssh kki
    cd /cluster/data/hg17/bed/blastzSelf
    mkdir axtChrom run.2
    cd run.2
    cat << '_EOF_' > runLavToAxt.sh
#!/bin/sh

BASE=/cluster/data/hg17/bed/blastzSelf
SEQ1_DIR=/cluster/bluearc/scratch/hg/gs.18/build35/bothMaskedNibs
SEQ2_DIR=/cluster/bluearc/scratch/hg/gs.18/build35/bothMaskedNibs

CHR=$1
OUT=axtChrom/$CHR.axt
cd ${BASE}/lav/${CHR}
for D in *.lav
do
    smallout=$D.axt
    lavToAxt  $D $SEQ1_DIR $SEQ2_DIR stdout \
	| axtDropSelf stdin stdout \
	| axtSort stdin $smallout
done
cat `ls -1 *.lav.axt | sort -g` > $BASE/$OUT
'_EOF_'
    # << keep emacs coloring happy
    chmod +x runLavToAxt.sh

    cat << '_EOF_' > gsub
#LOOP
./runLavToAxt.sh $(path1) {check out line ../axtChrom/$(path1).axt}
#ENDLOOP
'_EOF_'
    # << keep emacs coloring happy

    ls ../lav > chrList
    gensub2 chrList single gsub jobList
    para create jobList
    para try
    para push
    #	This is a tough load on eieio.  Managable, but the load should
    #	be monitored to make sure it isn't severe.  I saw about 100 to 150
    #	the chr19 job will not finish, even in parts it takes up too
    #	much memory and the node it runs on ends up swapping endlessly.
    #	Need to go to kolossus to do chr19
    para stop
    para recover jobList chr19JobList
    ssh kolossus
    cd /cluster/data/hg17/bed/blastzSelf/run.2
    time ./runLavToAxt.sh chr19
    #	real    43m14.797s
    #	user    12m56.670s
    #	sys     3m13.590s

    # translate sorted axt files into psl
    ssh eieio
    cd /cluster/data/hg17/bed/blastzSelf
    mkdir pslChrom
    set tbl = "blastzSelf"
    foreach f (axtChrom/chr*.axt)
      set c=$f:t:r
      echo "Processing chr $c"
      /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
    end
    #	That takes about 70 minutes

    # Load database tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastzSelf/pslChrom
    bash # if a csh/tcsh user
    for I in *.psl
    do
	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
	echo "done: ${I}"
    done
    # exit bash if you are tcsh
    #	This is an 80 minute job

    #	Check results
    #	featureBits hg17 blastzSelf
    #	252256266 bases of 2866216770 (8.801%) in intersection
    #	real    40m49.573s
    #	user    21m14.200s
    #	sys     2m10.420s

    #	featureBits hg16 blastzSelf
    #	254410837 bases of 2865248791 (8.879%) in intersection


# CHAIN SELF BLASTZ (DONE - 2004-07-07 - Hiram)

# The axtChain is best run on the small kluster, or the kk9 kluster
    ssh kki
    mkdir -p /cluster/data/hg17/bed/blastzSelf/axtChain/run1
    cd /cluster/data/hg17/bed/blastzSelf/axtChain/run1
    mkdir out chain

    ls -1S /cluster/data/hg17/bed/blastzSelf/axtChrom/*.axt > input.lst
    cat << '_EOF_' > gsub
#LOOP
doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} out/$(root1).out
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

#  May need -minScore=5000 for all chroms if chr19 won't finish on kolossus

    cat << '_EOF_' > doChain
#!/bin/csh
axtChain $1 /iscratch/i/gs.18/build35/bothMaskedNibs \
	/iscratch/i/gs.18/build35/bothMaskedNibs $2 > $3
'_EOF_'
    # << this line makes emacs coloring happy
    chmod a+x doChain

    # 46 jobs
    gensub2 input.lst single gsub jobList
    para create jobList
    para try
    para push # ... etc ...
# Completed: 45 of 46 jobs
# Crashed: 1 jobs
# CPU time in finished jobs:       8519s     141.98m     2.37h    0.10d  0.000 y
# IO & Wait Time:                  4795s      79.92m     1.33h    0.06d  0.000 y
# Average job time:                 296s       4.93m     0.08h    0.00d
# Longest job:                     2407s      40.12m     0.67h    0.03d
# Submission to last job:          3540s      59.00m     0.98h    0.04d

    #	chr19 did fail, on kolossus, try:
    ssh kolossus
    cd /cluster/data/hg17/bed/blastzSelf/axtChain/run1
    time axtChain /cluster/data/hg17/bed/blastzSelf/axtChrom/chr19.axt \
	/cluster/data/hg17/nib \
	/cluster/data/hg17/nib \
	chain/chr19.chain > out/chr19.out
    #	80 minute job, 1.5 Gb result:
    #	-rw-rw-r--    1 1588795432 Jul  7 21:54 chr19.chain

    # now on the file server, sort chains
    ssh eieio
    cd /cluster/data/hg17/bed/blastzSelf/axtChain
    time chainMergeSort run1/chain/*.chain > all.chain
    #	real    27m38.935s
    #	user    23m18.540s
    #	sys     2m39.300s
    #	A 5 Gb file:
    #	-rw-rw-r--    1 5267202936 Jul  7 22:23 all.chain


    time chainSplit chain all.chain
    #	real    29m27.062s
    #	user    22m48.250s
    #	sys     1m57.910s

    # optionally: rm run1/chain/*.chain

    # Load chains into database
    # next machine
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastzSelf/axtChain/chain
    bash	#	for tcsh users
    for I in *.chain
    do
        c=${I/.chain/}
        $HOME/bin/i386/hgLoadChain -normScore hg17 ${c}_chainSelf $I
        echo done $c
    done
    # exit bash if you are tcsh
    #	This is almost 3 hours to load

    ssh kolossus
    cd /cluster/data/hg17/bed/blastzSelf.2004-07-01
    time HGDB_CONF=~/.hg.conf.read-only featureBits \
	-noRandom -noHap hg17 chainSelfLink > fb.chainSelfLink 2>&1 &
    #	real    56m34.802s
    #	240976607 bases of 2851352871 (8.451%) in intersection

    #	featureBits hg17 chainSelf
    #	682833453 bases of 2866216770 (23.824%) in intersection
    #	featureBits hg16 chainSelf
    #	626345319 bases of 2865248791 (21.860%) in intersection

    #	DELIVER these chain files to hgdownload (2005-01-27 - Hiram)
    ssh eieio
    cd /cluster/data/hg17/bed/blastzSelf/axtChain/chain
    gzip chr*.chain

    ssh hgwdev
    mkdir /usr/local/apache/htdocs/goldenPath/hg17/vsSelf
    cd /cluster/data/hg17/bed/blastzSelf/axtChain/chain
    cp -p *.chain.gz /usr/local/apache/htdocs/goldenPath/hg17/vsSelf

    #	fixup README file, request push

# NET SELF (DONE - 2004-07-13 - Hiram)

    ssh eieio
    cd /cluster/data/hg17/bed/blastzSelf/axtChain
    mkdir preNet
    cd chain
    bash	#	for tcsh users
    for I in *.chain
    do
      echo preNetting $I
      /cluster/bin/i386/chainPreNet $I /cluster/data/hg17/chrom.sizes \
                        /cluster/data/hg17/chrom.sizes ../preNet/$I
    done
    #	23 minutes

    cd ..
    mkdir n1
    cd preNet
    for I in *.chain
    do
      N=${I/.chain/}.net
      echo primary netting $I
      /cluster/bin/i386/chainNet $I -minSpace=10 \
	/cluster/data/hg17/chrom.sizes /cluster/data/hg17/chrom.sizes \
	../n1/$N /dev/null
    done
    # exit bash if you are tcsh
    #	5 minute job

    cd ..
    cat n1/*.net | /cluster/bin/i386/netSyntenic stdin hNoClass.net
    #	memory usage 206442496, utime 3009 s/100, stime 252
    #	memory usage 2510467072, utime 19307 s/100, stime 3181

    ssh hgwdev
    cd /cluster/data/hg17/bed/blastzSelf/axtChain
    time netClass hNoClass.net hg17 hg17 human.net \
	-tNewR=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInHuman \
	-qNewR=/cluster/bluearc/scratch/hg/gs.18/build35/linSpecRep.notInHuman
    #	real    9m32.951s
    #	user    2m42.840s
    #	sys     1m23.460s

    # If things look good do
    ssh eieio
    cd /cluster/data/hg17/bed/blastzSelf/axtChain
    rm -r n1 hNoClass.net
    # Make a 'syntenic' subset of these with
    time netFilter -syn human.net > humanSyn.net
    #	real    0m29.851s
    #	user    0m27.200s
    #	sys     0m2.120s

    # Load the nets into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastzSelf/axtChain
    netFilter -minGap=10 human.net |  hgLoadNet hg17 netSelf stdin
    netFilter -minGap=10 humanSyn.net | hgLoadNet hg17 syntenyNetSelf stdin

    # check results
    # featureBits hg17 netSelf
    #	620827374 bases of 2866216770 (21.660%) in intersection
    # featureBits hg16 netSelf
    #	563788850 bases of 2865248791 (19.677%) in intersection
    # featureBits hg15 selfNet
    #	749177799 bases of 2866466359 (26.136%) in intersection


    # featureBits hg17 syntenyNetSelf
    #	404535376 bases of 2866216770 (14.114%) in intersection
    # featureBits hg16 syntenyNetSelf
    #	340871322 bases of 2865248791 (11.897%) in intersection

    # Add entries for net and chain to human/hg17 trackDb

    # make net
    ssh eieio
    cd /cluster/data/hg17/bed/blastzSelf/axtChain
    mkdir humanNet
    time netSplit human.net humanNet
    #	real    0m52.106s
    #	user    0m43.350s
    #	sys     0m5.170s

    # extract axts from net  - this should be combined with the sort and
    # maf conversion below
    mkdir ../axtNet 
    foreach n (humanNet/chr*.net)
	set c=$n:t:r
	echo "netToAxt: $c.net -> $c.axt"
	rm -f ../axtNet/$c.axt
	netToAxt humanNet/$c.net chain/$c.chain \
		/cluster/data/hg17/nib \
		/cluster/data/hg17/nib stdout > ../axtNet/$c.axt
	echo "Complete: $c.net -> axtNet/$c.axt"
    end

    # sort axt's and convert to maf format
    mkdir ../mafNet
    foreach f (../axtNet/chr*.axt)
        set c=$f:t:r
        echo $c.axt
        mv ../axtNet/$c.axt ../axtNet/$c.unsorted.axt
        axtSort ../axtNet/$c.unsorted.axt ../axtNet/$c.axt
        rm ../axtNet/$c.unsorted.axt
        axtToMaf ../axtNet/$c.axt \
            /cluster/data/hg17/chrom.sizes /cluster/data/hg17/chrom.sizes \
                ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=hg17.
    end
    # a 3 minute job

    XXXX - ! ! !   WE DO NOT NEED the Best and Tight tracks for Self ! ! !
    ssh hgwdev
    mkdir -p /cluster/data/hg17/bed/blastzSelf/axtBest
    cd /cluster/data/hg17/bed/blastzSelf/axtBest
    ln -s ../axtNet/chr*.axt .

    # copy net axt's to download area - XXX Do we need this for Self ?
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastzSelf/axtNet
    mkdir /usr/local/apache/htdocs/goldenPath/hg17/vsSelf
    cp -p *.axt /usr/local/apache/htdocs/goldenPath/hg17/vsSelf
    cd /usr/local/apache/htdocs/goldenPath/hg17/vsSelf
    nice gzip *.axt
    nice md5sum *.gz > md5sum.txt
    # add README.txt file to dir (use previous assembly's copy as template)

    #  Convert those axt files to psl
    ssh eieio
    cd /cluster/data/hg17/bed/blastzSelf
    mkdir pslBest
    foreach a (axtBest/chr*.axt)
	set c=$a:t:r
	echo "processing $c.axt -> ${c}_blastzBestSelf.psl"
    /cluster/bin/i386/axtToPsl axtBest/${c}.axt \
	S1.len S2.len pslBest/${c}_blastzBestSelf.psl
	echo "Done: ${c}_blastzBestSelf.psl"
    end

    # Load tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastzSelf/pslBest
    bash # if a csh/tcsh user
    for I in chr*BestSelf.psl
    do
	/cluster/bin/i386/hgLoadPsl -noTNameIx hg17 ${I}
	echo "done ${I}"
    done
    # exit bash if you are tcsh

    #	check results
    #	featureBits hg17 blastzBestSelf
    #	233978156 bases of 2866216770 (8.163%) in intersection
    #	featureBits hg16 blastzBestSelf
    #	225819219 bases of 2865248791 (7.881%) in intersection

# MAKING HUMAN AXTTIGHT FROM AXTBEST (NOT TO BE DONE - 2004-07-13 - Hiram)
    #	XXXX - ! ! !  DO NOT NEED axtBest for Self alignments
    #	Been done anyway, Robert and Gill like to see it.

# BLASTZ SELF CLEAN UP (DONE - 2004-07-15 - Hiram)
    ssh eieio
    cd /cluster/data/hg17/bed/blastzSelf
    nice rm -rf raw &
    nice rm axtChain/run1/chain/* &
    nice rm -fr axtChain/n1 axtChain/hNoClass.net &
    nice gzip axtChrom/* pslChrom/* lav/*/* axtChain/all.chain axtChain/*.net &


# CREATING BIG ZIPS (DONE - 2004-07-23 - Hiram)
    ssh eieio
    cd /cluster/data/hg17/jkStuff
    time ./zipAll.sh > zipAll.out 2>&1

    ssh hgwdev
    #  This stuff has to work in a different way because this stuff
    #	updates on a daily basis.
    cd /usr/local/apache/htdocs/goldenPath/hg17/bigZips
    featureBits hg17 refGene:upstream:1000 -fa=upstream1000.fa
    zip upstream1000.zip upstream1000.fa
    rm upstream1000.fa
    featureBits hg17 refGene:upstream:2000 -fa=upstream2000.fa
    zip upstream2000.zip upstream2000.fa
    rm upstream2000.fa
    featureBits hg17 refGene:upstream:5000 -fa=upstream5000.fa
    zip upstream5000.zip upstream5000.fa
    rm upstream5000.fa


# ENCODE REGIONS (DONE 2004-07-28 kate)

    ssh eieio
    cd /cluster/data/hg17/bed
    mkdir encodeRegions
    cd encodeRegions
    liftOver /cluster/data/hg16/bed/encodeRegions/encodeRegions.bed \
        /cluster/data/hg16/bed/liftOver/hg16ToHg17.over.chain \
                encodeRegions.bed encodeRegions.unmapped
    wc -l encodeRegions.*
    # 44 encodeRegions.bed
    # 0 encodeRegions.unmapped
    ssh hgwdev
    cd /cluster/data/hg17/bed/encodeRegions
    hgLoadBed hg17 encodeRegions encodeRegions.bed -noBin


# H-INVITATIONAL GENE ANNOTATION DATABASE (WORKING 2004-07-28 kate)
    # http://www.jbirc.aist.go.jp/hinv/top.html
    # Create knownGene table to reference HINV gene ID's
    #  for link on knownGenes details page
    # Also, create an HINV gene track

    # download CDNA file release 1.5 -- got release # from downloads page).
    ssh kksilo
    mkdir -p /cluster/data/hinv
    cd /cluster/data/hinv
    wget http://www.jbirc.aist.go.jp/hinv/download/alldata/flatfile/FCDNA.gz
    gunzip FCDNA.gz
    mv FCDNA FCDNA.1.5

    # set up assembly work area
    ssh eieio
    cd /cluster/data/hg17
    mkdir -p bed/hinv
    cd bed/hinv

    # extract H-INV ID's and Genbank accessions of mRNAs
    awk '/CDNA_ACCESSION-NO:/ {print $2}' < /cluster/data/hinv/FCDNA.1.5 \
                                                        > accessions.txt
    awk '/CDNA_H-INVITATIONAL-ID:/ {print $2}' < /cluster/data/hinv/FCDNA.1.5 \
                                                        > ids.txt
    paste accessions.txt ids.txt > queries.txt
    wc -l ids.txt
        #   41118 ids.txt

    # create PSL file from alignments for these mRNA's, extracted from the 
    #       table of all aligned mRNA's
    ssh hgwdev
    cd /cluster/data/hg17/bed/hinv
    hgsql hg17 -s -e "SELECT * FROM all_mrna"  | cut -f 2- > all_mrna.tab

    ssh eieio
    cd /cluster/data/hg17/bed/hinv
    pslReps /dev/null stdout /dev/null | cat - all_mrna.tab > all_mrna.psl
        # using pslReps to generate the PSL file header
    ~kate/bin/i386/pslSelect -queryPairs=queries.txt all_mrna.psl hinv_mrna.psl
        # NOTE: generated with pslSelect.c v1.3 (1.4 is broken -- test is
        # setup in hg/pslSelect/tests & I requested Robert take a look)

    # load track of mrna alignments
    hgwdev
    cd /cluster/data/hg17/bed/hinv
    hgLoadPsl hg17 -table=HInvGeneMrna hinv_mrna.psl
    hgsql hg17 -s -e \
        "select distinct(qName) from HInvGeneMrna order by qName" > hg17.mrna
    hgsql hg16 -s -e \
        "select distinct(qName) from HInvGeneMrna order by qName" > hg16.mrna
    wc -l hg*.mrna
        # 40998 hg16.mrna
        # 41023 hg17.mrna

    comm -1 -3 *.mrna > hg17.aligned
    wc -l hg17.aligned
        # 29 (transcripts newly aligned in hg17)
    comm -2 -3 *.mrna > hg16.aligned
    wc -l hg16.aligned
        # 4 (transcripts no longer aligned in hg17)
    comm -2 -3 ids.txt hg17.mrna > hg17.notaligned
    wc -l hg17.notaligned
        # 95 (transcripts not aligned in hg17 -- checking on why...)

    # also make a table with various useful items for each transcript
    ssh hgwdev
    hgsql hg17 < ~/kent/src/hg/lib/HInv.sql
    cd /cluster/data/hg17/bed/hinv
    /cluster/data/hinv/hinvToTable.pl < /cluster/data/hinv/FCDNA.1.5 > HInv.tab
    echo 'load data local infile "HInv.tab" into table HInv' | hgsql hg17
    hgsql hg16 -s -e "select count(*) from HInv"
        # 41118
    hgsql hg17 -s -e "select count(*) from HInv"
        # 41118

    # create table for knownGenes detail page
    ssh hgwdev
    cd /cluster/data/hg17/bed/hinv
    hgMapToGene hg17 HInvGeneMrna knownGene knownToHInv


# GENEID GENE PREDICTIONS (DONE 7/30/04 angie)
    ssh hgwdev
    mkdir /cluster/data/hg17/bed/geneid
    cd /cluster/data/hg17/bed/geneid
    foreach chr (`awk '{print $1;}' ../../chrom.sizes`)
      wget http://genome.imim.es/genepredictions/H.sapiens/golden_path_200405/geneid_v1.2/$chr.gtf
      wget http://genome.imim.es/genepredictions/H.sapiens/golden_path_200405/geneid_v1.2/$chr.prot
    end
    # Add ".1" suffix to each item in .prot's, to match transcript_id's in gtf
    cp /dev/null geneid.fa
    foreach f (chr*.prot)
      perl -wpe 's/^(>chr\S+)/$1.1/' $f >> geneid.fa
    end
    ldHgGene -gtf -genePredExt hg17 geneid *.gtf 
    hgPepPred hg17 generic geneidPep geneid.fa


# MITOPRED DATA FOR HGGENE (DONE 7/30/04 angie)
    ssh hgwdev
    mkdir /cluster/data/hg17/bed/mitopred
    cd /cluster/data/hg17/bed/mitopred
    wget http://mitopred.sdsc.edu/data/hum_30.out
    perl -wpe 's/^(\S+)\s+\S+\s+(.*)/$1\t$2/' hum_30.out > mitopred.tab
    cat > mitopred.sql << '_EOF_'
# Prediction of nuclear-encoded mito. proteins from http://mitopred.sdsc.edu/
CREATE TABLE mitopred (
    name varchar(10) not null,      # SwissProt ID
    confidence varchar(8) not null, # Confidence level
              #Indices
    PRIMARY KEY(name(6))
);
'_EOF_'
    # << this line makes emacs coloring happy
    hgsql hg17 < mitopred.sql
    hgsql hg17 -e 'load data local infile "mitopred.tab" into table mitopred'


# NUCLEAR PROTEIN DATABASE (IN PROGRESS 7/30/04 angie)
    ssh eieio
    mkdir /cluster/data/hg17/bed/npd
    cd /cluster/data/hg17/bed/npd
    wget ftp://ftp.hgu.mrc.ac.uk/pub/npd/database.zip
    unzip database.zip
    # OK, it's one big .mdb (Microsoft Access DB) file.  
    # Googling... can buy a converter for $40... free trial .exe...


# CREATING REFFULL - DBTSS MRNA (DONE - 2004-08-02 - Hiram)

    ssh to eieio
    mkdir /cluster/data/hg17/bed/refFull
    cd /cluster/data/hg17/bed/refFull
    wget --timestamping "ftp://ftp.hgc.jp/pub/hgc/db/dbtss/ref-full.fa.gz" .
    wget --timestamping "ftp://ftp.hgc.jp/pub/hgc/db/dbtss/readme" .
    #	See also: http://dbtss.hgc.jp/index.html

    #	gunzip it and split the ref-rull.fa file into about 200 pieces
    #	(faSplit won't do this job if it is:
    #	zcat ref-full.fa.gz | faSplit sequence stdin 50 splitRefFull
    gunzip ref-full.fa.gz
    faSplit sequence ref-full.fa 50 splitRefFull
    gzip ref-full.fa

    #	copy to Iservers
    ssh kkr1u00
    cd /cluster/data/hg17/bed/refFull
    mkdir /iscratch/i/gs.18/build35/refFull
    cp -p split*.fa /iscratch/i/gs.18/build35/refFull
    /cluster/bin/iSync
    #	no longer need these split files here
    rm -f split*.fa

    #	run alignments on kluster
    ssh kk
    cd /cluster/data/hg17/bed/refFull
    ls -1S /scratch/hg/gs.18/build35/maskedContigs > genome.lst
    ls -1S /iscratch/i/gs.18/build35/refFull > refFull.lst


    #	Use BLAT to generate refFull alignments as so:
    cat << '_EOF_' > gsub
#LOOP
/cluster/bin/i386/blat -ooc=/scratch/hg/h/11.ooc -q=dna -t=dna {check in exists /scratch/hg/gs.18/build35/maskedContigs/$(path1)} {check in exists+ /iscratch/i/gs.18/build35/refFull/$(path2)} {check out line+ psl/$(root1)/$(root2).psl}
#ENDLOOP
'_EOF_'
    # << keep emacs coloring happy

    bash # if a csh/tcsh user
    mkdir psl
    cat genome.lst | sed -e "s/.fa//" | while read C
	do
		mkdir psl/${C}
	done
    # exit bash if you are tcsh
    gensub2 genome.lst refFull.lst gsub jobList
    para create jobList
    #	18240 jobs written to batch
    para try
    para check
    para push ... etc ...
# Completed: 18240 of 18240 jobs
# CPU time in finished jobs:      37011s     616.85m    10.28h    0.43d  0.001 y
# IO & Wait Time:                 62630s    1043.84m    17.40h    0.72d  0.002 y
# Average job time:                   5s       0.09m     0.00h    0.00d
# Longest job:                       51s       0.85m     0.01h    0.00d
# Submission to last job:           850s      14.17m     0.24h    0.01d

    #	Process refFull alignments into near best in genome.
    ssh eieio
    cd /cluster/data/hg17/bed/refFull
    pslSort dirs raw.psl tmp psl/*
    pslReps -minCover=0.2 -sizeMatters -minAli=0.965 \
	-nearTop=0.001 raw.psl contig.psl /dev/null
    liftUp -nohead all_refFull.psl ../../jkStuff/liftAll.lft warn contig.psl
    pslSortAcc nohead chrom tmp all_refFull.psl
    pslCat -dir chrom > refFullAli.psl

    #	Load refFull alignments into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/refFull
    hgLoadPsl hg17 -tNameIx refFullAli.psl

# VAR_MULTIZ HG17/MM5/RN3/GALGAL2/FR1 (acs 2004-08-12)

# This is a new, experimental version of multiz written by Minmei at
# PSU and sent by e-mail from Webb.  This version allows for a
# progressive alignment strategy (i.e., alignment construction in a
# post-order traversal of the tree) using only pairwise alignments of
# each sequence with the reference sequence and without any need for
# "staging".  Here's a little blurb about it from the header of
# var_multiz.v3.c.

#  var_multiz.v3.c
#
#  Variant to multiz program. It aligns two files of
#  alignment blocks where top row is always the reference,
#  assuming blocks are increasing ordered based on the
#  start position on the refernece seqence. Single-coverage
#  on reference is required at this stage.
#
#  Four arguments are required: char* arg1, char* arg2,
#  int arg3, int arg4. arg1 and arg2 are two files need
#  to be aligned together. The alignment of reference in
#  two files are either fixed or not, determined from
#  argurments arg3 and arg4. arg3 and arg4 are either 1
#  or 0, but cannot be 1 at the same time. 1 means
#  reference is fixed. v1 and v2 cannot be both 1.
#  ...

    mkdir /cluster/data/hg17/bed/var_multiz.2004-08-12

    # unpack source and compile
    cp /cluster/home/acs/var_multiz.tar.gz /cluster/data/hg17/bed/var_multiz.2004-08-12
    cd /cluster/data/hg17/bed/var_multiz.2004-08-12
    tar xfz var_multiz.tar.gz
    cd var_multiz_source
    make

     NOTE (8/14): this version of the source is already out of date!
    # Source is now checked in under hg3rdParty and updated binaries
    # are being kept under /cluster/bin/penn/var_multiz

    # script for creating the 5-way alignments for a given chromosome
    # (acs, 8/20/04) below revised after e-mail exchange with Minmei
    cat << '_EOF_' > doVarMultiz.csh
#!/bin/csh -fe

set chr = $1			# may include _random or _hla_hap[12]

set REF = hg17.$chr
set RAT = /cluster/bluearc/hg17/multiz8way/rn3/$chr.maf
set MOUSE = /cluster/bluearc/hg17/multiz8way/mm5/$chr.maf
set CHICKEN = /cluster/bluearc/hg17/multiz8way/galGal2/$chr.maf
set FISH = /cluster/bluearc/hg17/multiz8way/fr1/$chr.maf
set DEST = /cluster/data/hg17/bed/var_multiz.2004-08-12/maf/$chr.maf

set VMZ = /cluster/bin/penn/var_multiz
set PROJECT = /cluster/bin/penn/var_multiz.2004.08.12/maf_project

mkdir -p $DEST:h

if ( -s $RAT && -s $MOUSE ) then 
    echo "Aligning $RAT $MOUSE..."
    $VMZ $RAT $MOUSE 0 0 > /scratch/$chr.tmp1.maf
    echo "Projecting on $REF..."
    $PROJECT /scratch/$chr.tmp1.maf $REF > /scratch/$chr.hrm.maf
else if ( -s $RAT ) then
    cp $RAT /scratch/$chr.hrm.maf
else if ( -s $MOUSE ) then
    cp $MOUSE /scratch/$chr.hrm.maf
endif

if ( -s $CHICKEN && -s /scratch/$chr.hrm.maf ) then
    echo "Adding $CHICKEN..."
    $VMZ /scratch/$chr.hrm.maf $CHICKEN 1 0 > /scratch/$chr.tmp2.maf
    echo "Projecting on $REF..."
    $PROJECT /scratch/$chr.tmp2.maf $REF > /scratch/$chr.hrmc.maf
else if ( -s $CHICKEN ) then
    cp $CHICKEN /scratch/$chr.hrmc.maf
else if ( -s /scratch/$chr.hrm.maf ) then
    cp /scratch/$chr.hrm.maf /scratch/$chr.hrmc.maf
endif 

if ( -s $FISH && -s /scratch/$chr.hrmc.maf ) then
    echo "Adding $FISH..."
    $VMZ /scratch/$chr.hrmc.maf $FISH 1 0 > /scratch/$chr.tmp3.maf
    echo "Projecting on $REF..."
    $PROJECT /scratch/$chr.tmp3.maf $REF > $DEST
else if ( -s $FISH ) then
    cp $FISH $DEST
else if ( -s /scratch/$chr.hrmc.maf ) then
    cp /scratch/$chr.hrmc.maf $DEST
endif 


echo "Done."
rm /scratch/$chr.tmp[123].maf /scratch/$chr.hrm.maf /scratch/$chr.hrmc.maf
'_EOF_'
    # << keep emacs coloring happy    
    chmod 755 doVarMultiz.csh

    for file in `find /cluster/bluearc/hg17/multiz8way/rn3 /cluster/bluearc/hg17/multiz8way/mm5 /cluster/bluearc/hg17/multiz8way/galGal2 /cluster/bluearc/hg17/multiz8way/fr1 -name "chr*.maf"` ; do echo `basename $file .maf` ; done | sort -u > chrlist
    rm -f jobs.lst
    for chr in `cat chrlist` ; do echo "doVarMultiz.csh $chr" >> jobs.lst ; done

    # run cluster job
    ssh kk ; cd /cluster/data/hg17/bed/var_multiz.2004-08-12; para create jobs.lst ; para try ; para push 
    # (etc.)

Completed: 46 of 46 jobs
CPU time in finished jobs:      71302s    1188.36m    19.81h    0.83d  0.002 y
IO & Wait Time:                  1162s      19.37m     0.32h    0.01d  0.000 y
Average job time:                1575s      26.26m     0.44h    0.02d
Longest job:                     6353s     105.88m     1.76h    0.07d
Submission to last job:          6362s     106.03m     1.77h    0.07d

    # for now just create an ordinary maf track (conservation later)
    rm -rf /gbdb/hg17/varMultiz/maf/varMultizMm5Rn3GalGal2Fr1
    mkdir -p /gbdb/hg17/varMultiz/maf/varMultizMm5Rn3GalGal2Fr1
    ln -s /cluster/data/hg17/bed/var_multiz.2004-08-12/maf/*.maf /gbdb/hg17/varMultiz/maf/varMultizMm5Rn3GalGal2Fr1
    /cluster/bin/i386/hgLoadMaf hg17 -warn varMultizMm5Rn3GalGal2Fr1 -pathPrefix=/gbdb/hg17/varMultiz/maf/varMultizMm5Rn3GalGal2Fr1
    chmod 775 /gbdb/hg17/varMultiz /gbdb/hg17/varMultiz/maf /gbdb/hg17/varMultiz/maf/varMultizMm5Rn3GalGal2Fr1 /cluster/data/hg17/bed/var_multiz.2004-08-12 /cluster/data/hg17/bed/var_multiz.2004-08-12/maf
    chmod 664 /cluster/data/hg17/bed/var_multiz.2004-08-12/maf/*.maf

    # trackDb entry
# track varMultizMm5Rn3GalGal2Fr1
# shortLabel varMultiz5Way
# longLabel Human/Mouse/Rat/Chicken/Fugu Var-Multiz 
# group compGeno
# priority 190
# visibility hide
# type maf


# elephant human blastz alignment by Robert Aug 11 2004
mkdir /cluster/bluearc/elephant
cd /cluster/bluearc/elephant

#get reads and qual scores from trace repository
for i in `cat trace.lst`; do echo $i ; wget ftp://ftp.ncbi.nih.gov/pub/TraceDB/loxodonta_africana/fasta.loxodonta_africana.$i.gz ; done
for i in `cat trace.lst`; do echo $i ; wget ftp://ftp.ncbi.nih.gov/pub/TraceDB/loxodonta_africana/qual.loxodonta_africana.$i.gz ; done
for i in `cat trace.lst` ; do zcat fasta.loxodonta_africana.$i.gz > loxodonta_africana.$i.fa ; done     

#trim reads
for i in `cat trace.lst` ; do nice gunzip -c qual.loxodonta_africana.$i.gz > qual.loxodonta_africana.$i; faTrimRead loxodonta_africana.$i.fa qual.loxodonta_africana.$i tmp.$i.fa lift.$i.lft; mv -f tmp.$i.fa loxodonta_africana.$i.fa ; rm -f qual.loxodonta_africana.$i ; done
for i in `cat trace.lst`; do faSize -detailed=on loxodonta_africana.$i.fa > mac.$i.len ; done  
cat mac.0*.len > S2.len
for i in `cat trace.lst`; do sed -e s/S2.len/mac.$i.len/ < DEF > DEF.$i ; done

#split fa reads into 10mb chunks for blastz run and distribute to i-servers.
ssh kkr1u00
for i in `cat trace.lst`; do nice faSplit about loxodonta_africana.$i.fa 10000000 /iscratch/i/elephant/${i}.mac. ; done
cd /iscratch/i/elephant
find split -name \*.fa > /cluster/bluearc/elephant/mac.lst
cd /cluster/bluearc/elephant
hgsql hg17 -N < chromLen.sql > S1.len
cd /iscratch/i/elephant
iSync

#setup cluster run to blastz reads to human genome
ssh kk
cd /cluster/bluearc/elephant
BlastZ_run0.sh
cd run.0
para create jobList
para push

#94798 jobs in batch
#149 jobs (including everybody's) in Parasol queue.
#Checking finished jobs
#Completed: 94797 of 94798 jobs
#Crashed: 1 jobs
#CPU time in finished jobs:   14183153s  236385.89m  3939.76h  164.16d  0.450 y
#IO & Wait Time:                310938s    5182.30m    86.37h    3.60d  0.010 y
#Average job time:                 153s       2.55m     0.04h    0.00d
#Longest job:                     1770s      29.50m     0.49h    0.02d
#Submission to last job:         52186s     869.77m    14.50h    0.60d

ssh kkr9
BlastZ_run1.sh
cd run.1
para create jobList
para push
#341 jobs in batch
#151 jobs (including everybody's) in Parasol queue.
#Checking finished jobs
#Completed: 341 of 341 jobs
#CPU time in finished jobs:     142914s    2381.91m    39.70h    1.65d  0.005 y
#IO & Wait Time:                 14078s     234.63m     3.91h    0.16d  0.000 y
#Average job time:                 460s       7.67m     0.13h    0.01d
#Longest job:                      782s      13.03m     0.22h    0.01d
#Submission to last job:          1954s      32.57m     0.54h    0.02d

#generate lst and fa files for each chromosome for faster lavToAxt
cd /cluster/bluearc/elephant
echo "select chrom from chromInfo;" > chrom.sql
hgsql hg17 -B -N < chrom.sql > chrom.lst

for i in `cat chrom.lst` ; do grep -h '>' lav/$i/* | awk '{print $1}' | sed -e 's/"//g' | sed -e 's/>//g' > mac.$i.lst ; echo $i ; done

ssh kki
cd /cluster/bluearc/elephant
mkdir -p splitChrom
/bin/rm splitChrom/*
gensub2 trace.lst chrom.lst gsub.split spec.split
para create spec.split
para push
#322 jobs in batch
#Checking finished jobs
#Completed: 322 of 322 jobs
#CPU time in finished jobs:        819s      13.65m     0.23h    0.01d  0.000 y
#IO & Wait Time:                  3278s      54.63m     0.91h    0.04d  0.000 y
#Average job time:                  13s       0.21m     0.00h    0.00d
#Longest job:                       51s       0.85m     0.01h    0.00d
#Submission to last job:           462s       7.70m     0.13h    0.01d


cd /cluster/bluearc/elephant/splitChrom
for i in `cat /cluster/bluearc/elephant/chrom.lst` ; do cat mac.*.$i.fa > mac.$i.fa ; echo $i ; done

#lav to axt run
ssh kk
cd /cluster/bluearc/elephant
mkdir -p run.2
#NOTE: chr19 must be run on kolossus with 64bit executables
#change SEQ1_DIR form /scratch to /iscratch for mini cluster
. DEF
echo "#LOOP" > run.2/gsub
echo '/cluster/bin/scripts/blastz-contiglav2axt '${BASE}'/lav/$(root1) {check out line+ '${BASE}'/axtChrom/$(root1).axt} '${SEQ1_DIR}' /cluster/bluearc/elephant/splitChrom/mac.$(root1).fa' >> run.2/gsub
echo "#ENDLOOP" >> run.2/gsub
cd run.2
gensub2 ../chrom.lst single gsub jobList
para create jobList
para push
#chrM has no data and crashed
#46 jobs in batch
#Checking finished jobs
#Completed: 45 of 46 jobs
#Crashed: 1 jobs
#CPU time in finished jobs:     249970s    4166.17m    69.44h    2.89d  0.008 y
#IO & Wait Time:                  5407s      90.11m     1.50h    0.06d  0.000 y
#Average job time:                5675s      94.58m     1.58h    0.07d
#Longest job:                    27065s     451.08m     7.52h    0.31d
#Submission to last job:         47744s     795.73m    13.26h    0.55d



#split reads by prefix so axtBest will fit in memory
mkdir axtByQ
cat mac*.lst | awk -F\| '{print substr($3,1,3)}' | sort -nu >prefix.lst
for i in `cat prefix.lst` ; do cat axtChrom/*.axt | axtFilter -qStartsWith=gnl\|ti\|$i stdin | axtSwap stdin S1.len S2.len stdout | axtSort stdin axtByQ/q$i.axt ; done
mkdir axtByQBest
#lots of memory needed for reciprocal best
ssh kolossus
cd /cluster/bluearc/elephant/axtByQ
for i in `ls *.axt` ; do axtBest -quiet i all stdout | axtSwap stdin ../S2.len ../S1.len ../axtByQBest/$i ; echo $i done ; done
cd ../cluster/bluearc/elephant/axtByQBest
cat q*.axt | axtSplitByTarget stdin .
mkdir axtRecipBest
for i in `cat chrom.lst` ; do axtSort axtByQBest/$i.axt stdout | axtBest stdin $i axtRecipBest/$i.axt ; echo $i ;done 
for i in `cat chrom.lst` ; do axtToMaf axtRecipBest/$i.axt S1.len S2.len maf/$i.maf -tPrefix=hg17. -qPrefix=rm1. -scoreZero ; done 
for i in `cat chrom.lst` ; do mafFilter -minScore=1000 maf/$i.maf > mafFilter/$i.maf ; done 
# record coverage
cd mafFilter
for i in `cat ../chrom.lst` ; do nice mafCoverage hg17 $i.maf -count=2 > $i.cov ; echo done $i ; done   

# CHIMP DELS FROM HG16 (DONE 2004-08-17 kate)
# NOTE: this track just for development -- it should be regenerated from the latest
#       alignments instead of lifted.
    ssh hgwdev
    cd /cluster/data/hg17/bed
    mkdir -p chimpDels
    cd chimpDels
    hgsql -s hg16 -e "SELECT * FROM chimpDels" | cut -f 2- > chimpDels.hg16.bed
    liftOver chimpDels.hg16.bed \
        /cluster/data/hg16/bed/liftOver/hg16ToHg17.over.chain \
                chimpDels.bed chimpDels.unmapped
    wc -l chimpDels.bed chimpDels.unmapped
    # 27662 chimpDels.bed
    # 132 chimpDels.unmapped
    # 27794 total
    hgLoadBed hg17 chimpDels chimpDels.bed -noBin

### CREATE chimpFixedDiff -- panTro1 (Daryl, August 18, 2005)

    # Convert chimp quality scores from uncompressed to compressed
    # chromosome format.  This took 22 minutes on crow.
    ## previously done for hg16
    # cd /cluster/data/panTro1
    # cat */chr*.qa | qaToQac stdin chrom.qac

    # Make single base pair high quality differences into a bed file
    # and load into database
    cd /cluster/data/hg17/bed
    mkdir chimpFixedDiff
    cd chimpFixedDiff
    sed 's/simpleNucDiff/chimpFixedDiff/' ~/kent/src/hg/lib/simpleNucDiff.sql > chimpFixedDiffs.sql

    # chimpHiQualDiffs was changed to allow different 
    #    quality parameters as command line options

    set axtDir = /cluster/data/hg17/bed/blastz.panTro1/axtRBestNet
    # This crashed twice at the same place, but ran successfully when
    # each chromosome was run separately.
    ##  time chimpFixedDiffs /$axtDir /cluster/data/panTro1/chrom.qac chimpFixedDiffs.bed >& chimpFixedDiffs.log
    mkdir chroms; cd chroms
    ls -1 $axtDir | grep chr | grep axt | sed 's/.axt//' | xargs mkdir
    # rmdir chr*random
    touch cfd.log
    foreach f (chr*)
	echo -n $f "  "
	ln -s /$axtDir/$f.axt $f/$f.axt
	time nice chimpFixedDiffs $f /cluster/data/panTro1/chrom.qac $f.chimpFixedDiffs.bed>>& cfd.log
    end
    rm ../chimpFixedDiffs.bed
    cat chr*bed > ../chimpFixedDiffs.bed

    ## The load (sort) ran out of memory on hgwdev, so I sorted the
    ## file first on kolossus (3 minutes) and then loaded it on hgwdev
    ssh kolossus
    hgLoadBed -strict -sqlTable=chimpFixedDiffs.sql -noLoad hg16 chimpFixedDiff chimpFixedDiffs.bed
    exit
    ## hgwdev (37 minutes)
    hgLoadBed -hasBin -noSort -sqlTable=chimpFixedDiffs.sql hg16 chimpFixedDiff bed.tab

    TODO: need to filter out polymorphic sites (SNPs)





#  Load firstEF track (DONE 2004-08-18 braney)
    ssh hgwdev
    mkdir -p /cluster/data/hg17/bed/firstEF
    cd /cluster/data/hg17/bed/firstEF
    wget "http://bioinformatics.med.ohio-state.edu/downloads/firstEFMay04.bed.gz"
    cat << '_EOF_' > sedScript
s/chr23/chrX/g
s/chr24/chrY/g
/^>/d
/^$/d
/^No/d
'_EOF_'

    zcat firstEFMay04.bed.gz | sed -f sedScript | awk  "{OFS=\"\t\"} {\$3 +=1; print  \$0}" > firstEF.bed
    hgLoadBed hg17 firstEF firstEF.bed
    rm firstEF.tab
    gzip *.bed
#done firstEF


# GENE BOUNDS (RNACLUSTER) (DONE 08-18-2004 Chuck)
# Create rnaCluster table (depends on {est,mrna}OrientInfo)
cd ~sugnet/store1/altSplice/hg17/
mkdir rnaCluster
cd rnaCluster/
mkdir chrom
# Create a list of accessions that come from RAGE libraries and need to be excluded.
~/kent/src/hg/geneBounds/clusterRna/generateRageAccList.csh hg17 rage.libs
foreach f (/cluster/data/hg17/nib/chr*.nib)
    set c = $f:t:r
    set out = chrom/$c.bed
    # Exclude accesions in the RAGE file
    echo clusterRna -mrnaExclude=hg17.rage.libs hg17 /dev/null $out -chrom=$c
    clusterRna -mrnaExclude=hg17.rage.libs hg17 /dev/null $out -chrom=$c
end
hgLoadBed hg17 rnaCluster chrom/*.bed
mkdir /cluster/data/hg17/bed/rnaCluster
cp -r chrom /cluster/data/hg17/bed/rnaCluster

#  miRNA track (CORRECTION 2004-12-09 - Hiram)
    #	Received the following correction from Michel Weber:
    #	Could you please replace the two lines:
    #	chr6 72169974 72170045 hsa-mir-30a 480 - 72169977 72169999
    #	chr6 72170017 72170040 hsa-mir-30a-5p 480 - 72170017 72170040
    #	by:
    #	chr6 72169974 72170045 hsa-mir-30a 480 - 72169977 72169999
    #	chr6 72169974 72170045 hsa-mir-30a 480 - 72170017 72170040
    #	(The first line remains identical, only the second is changed. The 
    #	repetition of the hsa-mir-30a entry means that both strands of its
    #	hairpin structure are matured into microRNAs, named hsa-miR-30a-3p and 
    #	hsa-miR-30a-5p in Rfam database).
    ssh hgwdev
    cd /cluster/data/hg17/bed/miRNA
    mv miRNA_hg17_1.bed miRNA_hg17_1.bed.0
    cp miRNA_hg17_1.bed.0 miRNA_hg17_1.bed
    #	edit miRNA_hg17_1.bed to change the single line.  Then:
    mv hg17.bed hg17.bed.0
    egrep -v "^track |^browser " miRNA_hg17_1.bed | \
	sed -e "s/miR/mir/g; s/ sa-mir/ hsa-mir/g; s/ /\t/g;" > hg17.bed
    #	Check that the edit is in place properly:
    diff hg17.bed.0 hg17.bed
    #	and load it
    hgLoadBed hg17 miRNA hg17.bed
    #	Loaded 221 elements of size 8
    #	featureBits remains the same:
    featureBits hg17 miRNA
    #	18052 bases of 2866216770 (0.001%) in intersection

#  miRNA track (DONE 2004-09-03 - Hiram)(CORRECTED, see above 2004-12-09)
    #	The source data for this was received via email from Sam
    #	Griffiths-Jones to Donna 16 August 2004.  In other email Michel
    #	Weber asked to add one more data line to that file.

    #	data from: Sam Griffiths-Jones <sgj@sanger.ac.uk>
    #	and Michel.Weber@ibcg.biotoul.fr
    #	notify them if this assembly updates to renew this track
    cd /cluster/data/hg17/bed
    mkdir miRNA
    cd miRNA
    #	one name was missing the h in hsa-mir and one was miR instead of
    #	mir
    egrep -v "^track |^browser " miRNA_hg17_1.bed | \
	sed -e "s/miR/mir/g; s/ sa-mir/ hsa-mir/g; s/ /\t/g;" > hg17.bed
    hgLoadBed hg17 miRNA hg17.bed
    #	compare with previous results, should be relatively similar
    #	featureBits hg16 miRNA
    #	16923 bases of 2865248791 (0.001%) in intersection
    #	featureBits hg17 miRNA
    #	18052 bases of 2866216770 (0.001%) in intersection

    # entry is already in trackDb/trackDb.ra


## blastz mRNA track for internal use - Robert 8/12/04
mkdir /cluster/bluearc/hg17/mrnaBlastz
cd /cluster/bluearc/hg17/mrnaBlastz
/cluster/data/genbank/bin/i386/gbGetSeqs -gbRoot=/cluster/data/genbank genbank mrna mrna.fa -db=hg -native
mkdir -p split
faTrimPolyA mrna.fa trim.fa
faSplit about trim.fa 1000000 split/mrna
cp -ip trim.fa /panfs/ucsc.edu/home/scratch/hg17/mrnaBlastz
faSize trim.fa -detailed=on > S2.len
hgsql hg16 < chromInfo.sql > S1.len

BlastZ_run0.sh
cd run.0
para push
para time
#113894 jobs in batch
#207911 jobs (including everybody's) in Parasol queue.
#Checking finished jobs
#Completed: 113894 of 113894 jobs
#CPU time in finished jobs:   14423845s  240397.41m  4006.62h  166.94d  0.457 y
#IO & Wait Time:                334352s    5572.54m    92.88h    3.87d  0.011 y
#Average job time:                 130s       2.16m     0.04h    0.00d
#Longest job:                    38301s     638.35m    10.64h    0.44d
#Submission to last job:         59841s     997.35m    16.62h    0.69d

mkdir run.1
~angie/hummus/do.out2lav DEF > run.1/j
cd run.1
para create j
para push
para time
#341 jobs in batch
#208550 jobs (including everybody's) in Parasol queue.

#Checking finished jobs
#Completed: 341 of 341 jobs
#CPU time in finished jobs:      28990s     483.17m     8.05h    0.34d  0.001 y
#IO & Wait Time:                 43139s     718.98m    11.98h    0.50d  0.001 y
#Average job time:                 212s       3.53m     0.06h    0.00d
#Longest job:                     2015s      33.58m     0.56h    0.02d
#Submission to last job:          2187s      36.45m     0.61h    0.03d

#!/bin/tcsh
set base="/cluster/bluearc/hg17/mrnaBlastz"
    cd $base
    mkdir -p pslRaw
    foreach c (lav/*)
      pushd $c
      set chr=$c:t
      set out=$base/pslRaw/$chr.psl
      echo "Translating $chr lav to $out"
      cat `ls -1 *.lav | sort -g` \
        | lavToPsl stdin stdout \
        | sed -e 's@scratch/hg/gs.18/build35/bothMaskedNibs//@@' | sed -e 's/\.nib:[0-9]*-[0-9]*//' > $out
      popd
    end

mkdir run.2

for i in `awk '{print $1}' S1.len` ; do echo doSortFilter.sh ../pslRaw/$i.psl ../pslFilter/$i.psl  >> run.2/spec.dup ; done
cd run.2
para create spec.dup
para push
#46 jobs in batch
#3 jobs (including everybody's) in Parasol queue.
#Checking finished jobs
#Completed: 46 of 46 jobs
#CPU time in finished jobs:       4409s      73.48m     1.22h    0.05d  0.000 y
#IO & Wait Time:                  1082s      18.04m     0.30h    0.01d  0.000 y
#Average job time:                 119s       1.99m     0.03h    0.00d
#Longest job:                     3842s      64.03m     1.07h    0.04d
#Submission to last job:          3842s      64.03m     1.07h    0.04d
cd ..
for i in `awk '{print $1}' S1.len` ; do echo axtChain -linearGap=linearGap.txt -psl pslFilter/$i.psl /scratch/hg/gs.18/build35/bothMaskedNibs/ -faQ /panfs/ucsc.edu/home/scratch/hg17/mrnaBlastz/trim.fa chain/$i.chain >> spec.chain ; done
para create spec.chain
para push

cd run.3
para create spec.filter
para push
cd ..
ls /cluster/data/hg17/nib/*.nib > S1.lst
#Skip chainPreNet it is not good for mrna
#mkdir -p preNet
#
#cd chainFilter
#foreach i ( *.chain)
#chainPreNet $i ../S1.len ../S2.len ../preNet/$i
#end

mkdir run.4
cd run.4
for i in `awk '{print $1}' ../S1.len`; do echo "chainToPsl ../chainFilter/$i.chain ../S1.len ../S2.len ../S1.lst /panfs/ucsc.edu/home/scratch/hg17/mrnaBlastz/trim.fa ../psl/$i.psl" >> spec.chain2psl.new  ; done

pslCat psl/*psl > mrnaBlastz.psl
hgLoadPsl hg17 mrnaBlastz.psl

cp trim.fa /cluster/data/hg17/bed/mrnaBlastz/hg17Mrna.fa
ln /cluster/data/hg17/bed/mrnaBlastz/hg17Mrna.fa /gbdb/hg17/mrnaBlastz/ -s

hgLoadSeq -prefix=bz hg17 /gbdb/hg17/mrnaBlastz/hg17Mrna.fa

## end of blastz Mrna track

#### BUILD RETROGENE TRACK (done Robert 8/26/2004)
#### REBUILD RETROGENE TRACK (done Robert 12/24/2004 - but no notes - kuhn)
# diffs before push to beta:
#  1640 hg17.pseudoGeneLink.devOnly
#  9639 hg17.pseudoGeneLink.betaOnly
# 15091 hg17.pseudoGeneLink.common

#  RETROGENE TRACK data update - Robert - 2005-04-08
   (added by Jen 2006-01-31)
   - pushQ entry did not include psuedoMrna table. Old table is still
     present on RR. New data has since been lost on dev. 
     User impact: ~1000 sequence missing links in browser 
   - new all.joiner rule needed to link psuedoMrna to pseudoGeneLink table
   - current all.joiner rule between knownGene to pseudoGeneLink gives errors.
     the data types appear to be mismatched. pseudoGeneLink.kgName is
     a gene symbol, not the same identifier as in knownGene.name
   - data is to be regenerated soon and errors corrected at that time


mkdir /cluster/data/hg17/bed/pseudo
cd /cluster/data/hg17/bed/pseudo
ls /cluster/data/hg17/nib/*.nib > S1.lst
hgsql hg17 -N -B < allMrna.sql > allMrna.lst
cp  /cluster/data/genbank/data/aligned/genbank.142.0/hg17/full/mrna.native.psl.gz .
gunzip mrna.native.psl.gz
awk  '{OFS="\t";print $1,$2,$3,$4,$5,$6,$7,$8,$9,substr($10,1,index($10,".")-1),$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23}' mrna.native.rawPsl  > mrnaBlat.psl

hgsql hg17 -N -B < refGene.sql > refGene.tab
hgsql hg17 -B -N < mgcGene.sql > mgcGene.tab
cat ../../*/*.fa.out | awk '$5~/chr*/{OFS="\t";print $5,$6,$7}' >rmsk.bed

cd /cluster/bluearc/hg17/mrnaBlastz/
zcat /cluster/data/hg17/bed/blastz.mm5/axtChain/mouseSyn.net.gz | netToBed stdin mouseSyn.bed

hgsql hg17 < mrna.sql | grep -v matches | awk '{OFS="\t"; print $2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22}' > all_mrna.psl
ssh eieio
pslCat -nohead -check all_mrna.psl /cluster/bluearc/hg17/mrnaBlastz/psl/*.psl |awk '{print $0, $1*3-$2}' | sort -k 10,10 -k 22nr -T /tmp | awk '{OFS="\t"; print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21}' >  blatBlastzHg17.psl 
ssh hgwdev
cp blatBlastzHg17.psl /scratch/

tawk '$12 > 1 && $12 < 9999999{x=$11;$11=$12;$12=x;print $0}'  /cluster/data/kgDB/bed/hg17/kgBestMrna/sortedKnownGene.tab > sortedKnownGene.tab
#copy files to iServers for cluster run
ssh kkr1u00
/cluster/home/baertsch/bin/i386/pslSplit nohead -chunkSize=121 /iscratch/i/gs.18/build35/pseudo blatBlastzHg17.psl
cd /cluster/data/hg17/bed/pseudo
cp refGene.tab /iscratch/i/gs.18/build35/pseudo
cp /cluster/data/hg17/bed/simpleRepeat/simpleRepeat.bed /iscratch/i/gs.18/build35/pseudo
cp mrnaHg17.fa /iscratch/i/gs.18/build35/pseudo
cp sortedKnownGene.tab /iscratch/i/gs.18/build35/pseudo
cp rmsk.bed /iscratch/i/gs.18/build35/pseudo
cp all_mrna.psl /iscratch/i/gs.18/build35/pseudo
cp mouseSyn.bed /iscratch/i/gs.18/build35/pseudo
for i in `ls tmp*` ; do echo "doBuildkk.sh ${i%%.psl}" ; done | sed -e 's/tmp//g'  > ~/hg17/pseudo/spec.kk   
cd /iscratch/i/hg/gs.18/build35/pseudo
iSync

para create spec.kk
para push

#post process

# run from eieio
BLUE=/cluster/bluearc/hg17/pseudo
echo catting output
cat $BLUE/pseudoGeneLink[0-9]*.bed | sort -k1,1 -k2,3n >pseudoGeneLinkSort.bed ; /bin/rm $BLUE/pseudoGeneLink[0-9]*.bed
cat $BLUE/pseudo[0-9]*.psl > pseudo.psl ; /bin/rm $BLUE/pseudo[0-9]*.psl &
echo Filtering pseudoGeneLinkSort.bed
tawk '$5 > 10 && $15 > 10000 && $35 > 650 {OFS="\t";print $0}' pseudoGeneLinkSort.bed > pseudoGeneLinkSortFilter.bed
echo Removing Overlaps
doSplit
cd /cluster/bluearc/hg17/pseudo/run.o 
spec.overlap
cd ~/hg17/pseudo
cat /cluster/bluearc/hg17/pseudo/chr*pseudoNoOverlap.bed > pseudoGeneLinkNoOverlap.bed
echo Making psl
awk '{printf("%s\t%s\t%s\n", $5,$2,$3)}' pseudoGeneLinkNoOverlap.bed > pseudoGeneLinkSelect.tab
## 350 is the sacrad magic number and will probably change
tawk '$6>=350{print $0}' pseudoGeneLinkNoOverlap.bed > pseudoGeneLinkNoOverlapFilter.bed                       

pslSelect -qtStart=pseudoGeneLinkSelect.tab pseudo.psl pseudoMrna.psl
echo Loading Bed
hgLoadBed hg17 pseudoGeneLink pseudoGeneLinkNoOverlapFilter.bed -hasBin -sqlTable=/cluster/home/baertsch/kent/src/hg/lib/pseudoGeneLink.sql
echo Loading Psl
hgLoadPsl hg17 pseudoMrna.psl

## end of retroGene track


# 3-WAY MULTIZ MULTIPLE ALIGNMENT (MM5, RN3) (DONE 2004-08-27 kate)
#       HMR Maf's needed for regulatory potential track
    ssh eieio
    set multizDir = multiz.2004-08-27
    cd /cluster/data/hg17/bed/$multizDir
    set workingDir = /cluster/bluearc/hg17/$multizDir
    ln -s $workingDir /cluster/bluearc/hg17/multiz3way
    ln -s $multizDir multiz3way
    mkdir -p $workingDir
    mkdir -p /cluster/data/hg17/bed/$multizDir

# wrapper script for multiz
    # NOTE: first arg is pairwise, 2nd arg is multiple (to add to) 
    # NOTE: next time, modify script so it only needs one arg -- saves the
    # multiple dirname in a file for use by the next run
    cat << 'EOF' > doMultiz.csh
#!/bin/csh -fe
mkdir -p $3:h
/cluster/bin/penn/multiz $1 $2 - > $3
'EOF'

# << for emacs
    cat << 'EOF' > gsub
#LOOP
../doMultiz.csh {check in line /cluster/bluearc/hg17/multiz.2004-08-27/$(dir1)/$(root2).maf} {check in line /cluster/bluearc/hg17/multiz.2004-08-27/$(root1)/$(root2).maf} {check out line+ /cluster/bluearc/hg17/multiz.2004-08-27/$(root1)$(dir1)/$(root2).maf}
#ENDLOOP
'EOF'
# << for emacs

    chmod +x doMultiz.csh

    ssh eieio
    set workingDir = /cluster/bluearc/hg17/multiz.2004-08-27

    # copy mafs to bluearc -- chimp
    mkdir $workingDir/mm5
    cp /cluster/data/hg17/bed/blastz.mm5/mafNet/*.maf \
                $workingDir/mm5
    ls $workingDir/mm5/*.maf > chrom.lst

    # rat
    mkdir $workingDir/rn3
    cp /cluster/data/hg17/bed/blastz.rn3/mafNet/chr*.maf $workingDir/rn3

    # multiz - add in rn3 rat to human/mouse
    # 
    ssh kki
    set multizDir = multiz.2004-08-27
    set workingDir = /cluster/bluearc/hg17/$multizDir
    cd /cluster/data/hg17/bed/$multizDir
    mkdir run.rn3
    cd run.rn3
    echo "rn3/mm5" > species.lst
    gensub2 species.lst ../chrom.lst ../gsub jobList
    para create jobList
        # 47 jobs
    para try, check, push, check

    # copy 3-way mafs to build directory
    ssh eieio
    set multizDir = multiz.2004-08-27
    set workingDir = /cluster/bluearc/hg17/$multizDir
    ln -s $workingDir/mm5rn3 $workingDir/maf
    cd /cluster/data/hg17/bed/multiz.2004-08-27
    mkdir maf
    cp $workingDir/maf/*.maf maf


# BLASTZ TETRAODON (tetNig1) (DONE, 2004-08-26, hartera)
    # blastz requires lineage-specific repeats
    # Treat all repeats as lineage-specific.
    ssh kkr1u00
    mkdir /iscratch/i/gs.18/build35/linSpecRep.notInTetraodon
 foreach f (/iscratch/i/gs.18/build35/rmsk/chr*.fa.out)
 cp -p $f /iscratch/i/gs.18/build35/linSpecRep.notInTetraodon/$f:t:r:r.out.spec
    end

    mkdir /iscratch/i/tetNig1/linSpecRep.notInHuman
    foreach f (/iscratch/i/tetNig1/rmsk/chr*.fa.out)
      cp -p $f /iscratch/i/tetNig1/linSpecRep.notInHuman/$f:t:r:r.out.spec
    end
    iSync

    ssh kk
    mkdir -p /cluster/data/hg17/bed/blastz.tetNig1.2004-08-20
    ln -s /cluster/data/hg17/bed/blastz.tetNig1.2004-08-20 \
            /cluster/data/hg17/bed/blastz.tetNig1
    cd /cluster/data/hg17/bed/blastz.tetNig1
 
    # abridge repeats.
    # Treat all repeats as lineage-specific.
    cat << '_EOF_' > DEF
# human vs. Tetraodon
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin

# use same parameters as for danRer1-fr1
ALIGN=blastz-run
BLASTZ=blastz
BLASTZ_H=2000
BLASTZ_ABRIDGE_REPEATS=1

# TARGET: Human
SEQ1_DIR=/iscratch/i/hg17/bothMaskedNibs
SEQ1_RMSK=
SEQ1_FLAG=
SEQ1_SMSK=/iscratch/i/hg17/linSpecRep.notInTetraodon
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Tetraodon
SEQ2_DIR=/iscratch/i/tetNig1/nib
SEQ2_RMSK=
SEQ2_FLAG=
SEQ2_SMSK=/iscratch/i/tetNig1/linSpecRep.notInHuman
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/blastz.tetNig1

DEF=$BASE/DEF
RAW=$BASE/raw
CDBDIR=$BASE
SEQ1_LEN=$BASE/S1.len
SEQ2_LEN=$BASE/S2.len

#DEBUG=1
'_EOF_'
    # << this line keeps emacs coloring happy
    # save the DEF file in the current standard place
    chmod +x DEF
    cp DEF ~angie/hummus/DEF.hg17-tetNig1.2004-08-20
 
    # make sure BlastZ_run0.sh, BlastZ_run1.sh and BlastZ_run2.sh scripts
    # are in /cluster/data/hg17/jkStuff
    # edit BlastZ_run0.sh so directory for blastz is /cluster/bin/penn

    bash # if a csh/tcsh user
    . ./DEF
    /cluster/data/hg17/jkStuff/BlastZ_run0.sh
    cd run.0
    # check batch looks ok then
    para try, check, push, check, ....
# para time
# Completed: 19437 of 19437 jobs
# CPU time in finished jobs:    3225816s   53763.60m   896.06h   37.34d  0.102 y
# IO & Wait Time:                174096s    2901.60m    48.36h    2.01d  0.006 y
# Average job time:                 175s       2.92m     0.05h    0.00d
# Longest job:                      709s      11.82m     0.20h    0.01d
# Submission to last job:          5324s      88.73m     1.48h    0.06d

    # second cluster run: lift raw alignments -> lav dir
    ssh kki
    cd /cluster/data/hg17/bed/blastz.tetNig1
    bash # if a csh/tcsh user
    . ./DEF
    /cluster/data/hg17/jkStuff/BlastZ_run1.sh
    cd run.1
    para try, check, push, check etc.
# para time
# Completed: 341 of 341 jobs
# CPU time in finished jobs:        280s       4.66m     0.08h    0.00d  0.000 y
# IO & Wait Time:                  2183s      36.39m     0.61h    0.03d  0.000 y
# Average job time:                   7s       0.12m     0.00h    0.00d
# Longest job:                       41s       0.68m     0.01h    0.00d
# Submission to last job:           469s       7.82m     0.13h    0.01d

    # third run: lav -> axt
    ssh kki
    cd /cluster/data/hg17/bed/blastz.tetNig1
    mkdir axtChrom run.2
    cd run.2
    cat << '_EOF_' > do.csh
#!/bin/csh -ef
cd $1
cat `ls -1 *.lav | sort -g` \
| lavToAxt stdin /iscratch/i/gs.18/build35/bothMaskedNibs \
/iscratch/i/tetNig1/nib stdout \
| axtSort stdin $2
'_EOF_'
    # << this line makes emacs coloring happy
    chmod a+x do.csh
    cat << '_EOF_' > gsub
#LOOP
./do.csh {check in exists $(path1)} {check out line+ /cluster/data/hg17/bed/blastz.tetNig1/axtChrom/$(root1).axt}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy
    \ls -1Sd ../lav/chr* > chrom.list
    gensub2 chrom.list single gsub jobList
    wc -l jobList
    head jobList
    para create jobList
    para try, check, push, check,...
# para time
# Completed: 45 of 46 jobs
# Crashed: 1 jobs
# CPU time in finished jobs:         52s       0.87m     0.01h    0.00d  0.000 y
# IO & Wait Time:                   256s       4.27m     0.07h    0.00d  0.000 y
# Average job time:                   7s       0.11m     0.00h    0.00d
# Longest job:                       36s       0.60m     0.01h    0.00d
# Submission to last job:           275s       4.58m     0.08h    0.00d

# one job crashed because chr6_hla_hap1.axt is empty. Checked by running this
# again and then looked at the lav file which has no alignments in it.
                                                                                
   # translate sorted axt files into psl
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.tetNig1
    mkdir -p pslChrom
    set tbl = "blastzTetNig1"
    foreach f (axtChrom/chr*.axt)
      set c=$f:t:r
      echo "Processing chr $c"
      /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
    end

    # Load database tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.tetNig1/pslChrom
    foreach f (./*.psl)
      /cluster/bin/i386/hgLoadPsl -noTNameIx hg17 $f
      echo "$f Done"
    end

# original blastzTetNig1: 
# BLASTZ_H=2000
# BLASTZ_Y=3400
# BLASTZ_L=6000
# BLASTZ_K=2200
# BLASTZ_Q=/cluster/data/blastz/HoxD55.q
# BLASTZ_ABRIDGE_REPEATS=1
# featureBits -chrom=chr1 hg17 blastzTetNig1
# 6378680 bases of 222827847 (2.863%) in intersection
# featureBits -chrom=chr1 hg17 refGene:cds blastzTetNig1 -enrichment
# refGene:cds 1.246%, blastzTetNig1 2.863%, both 0.856%, cover 68.70%, 
# enrich 24.00x
# featureBits -chrom=chr1 hg17 refGene:cds blastzDanRer1 -enrichment
# refGene:cds 1.246%, blastzDanRer1 3.934%, both 0.831%, cover 66.72%, 
# enrich 16.96x
# comparable to zebrafish so good
# try same parameters with L=8000
# featureBits -chrom=chr1 hg17 refGene:cds blastzTetNig1L8k -enrichment
# refGene:cds 1.246%, blastzTetNig1L8k 2.095%, both 0.753%, cover 60.47%, 
# enrich 28.87x
# load chr1 with blastz using just H=2000 and default parameters
# featureBits -chrom=chr1 hg17 refGene:cds blastzTetNig1Default -enrichment
# refGene:cds 1.246%, blastzTetNig1Default 1.630%, both 0.808%, cover 64.87%, 
# enrich 39.80x
# rows in chr1_blastzTetNig1 tables 
# blastzTetNig1	95156
# blastzTetNig1L8k 58015
# blastzTetNig1Default 71342
# The default values also used for danRer1 vs fugu give good coverage and
# higher enrichment than blastzTetNig1 with less alignments so this will be
# used for the blastz track - now called blastzTetNig1.

# CHAIN TETRAODON (tetNig1) BLASTZ (DONE, 2004-08-26, hartera)
# Make chains with rescored blastz
    # Run axtChain on little cluster
    ssh kki
    cd /cluster/data/hg17/bed/blastz.tetNig1
    mkdir -p axtChain/run1
    cd axtChain/run1
    mkdir out chain
    ls -1S /cluster/data/hg17/bed/blastz.tetNig1/axtChrom/*.axt \
        > input.lst
cat << '_EOF_' > gsub
#LOOP
doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy

    # Make our own linear gap file with reduced gap penalties, 
    # in hopes of getting longer chains - works well for species at 
    # chicken-human distance or greater
    cat << '_EOF_' > ../../chickenHumanTuned.gap
tablesize	11
smallSize	111
position	1	2	3	11	111	2111	12111	32111	72111	152111	252111
qGap	325	360	400	450	600	1100	3600	7600	15600	31600	56600
tGap	325	360	400	450	600	1100	3600	7600	15600	31600	56600
bothGap	625	660	700	750	900	1400	4000	8000	16000	32000	57000
'_EOF_'
    # << this line makes emacs coloring happy

cat << '_EOF_' > doChain
#!/bin/csh
axtChain -linearGap=../../chickenHumanTuned.gap $1 \
    /iscratch/i/gs.18/build35/bothMaskedNibs \
    /iscratch/i/tetNig1/nib $2 >& $3
'_EOF_'
    # << this line makes emacs coloring happy
    chmod a+x doChain
    gensub2 input.lst single gsub jobList
    para create jobList
    para try, check, push, check...
# para time
# Completed: 45 of 46 jobs
# Crashed: 1 jobs
# CPU time in finished jobs:        553s       9.22m     0.15h    0.01d  0.000 y
# IO & Wait Time:                   102s       1.69m     0.03h    0.00d  0.000 y
# Average job time:                  15s       0.24m     0.00h    0.00d
# Longest job:                       56s       0.93m     0.02h    0.00d
# Submission to last job:           985s      16.42m     0.27h    0.01d

# one job crashed since chr6_hla_hap1.axt is empty - no alignments

   # now on the cluster server, sort chains
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
    chainMergeSort run1/chain/*.chain > all.chain
    chainSplit chain all.chain
   
    # take a look at score distr's
    foreach f (chain/*.chain)
      grep chain $f | awk '{print $2;}' | sort -nr > /tmp/score.$f:t:r
      echo $f:t:r >> hist5000.out
      textHistogram -binSize=5000 /tmp/score.$f:t:r >> hist5000.out
      echo ""
    end

    # only chr19 has a very large number of chains with score < 5000
    # load chr 1 into table
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain/chain
    hgLoadChain hg17 chr1_chainTetnig1 chr1.chain

# featureBits -chrom=chr1 hg17 refGene:cds chainTetnig1Link -enrichment
# refGene:cds 1.246%, chainTetnig1Link 1.582%, both 0.805%, cover 64.59%, 
# enrich 40.83x

    # try filtering with minScore of 5000
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
    mv all.chain all.chain.unfiltered
    chainFilter -minScore=5000 all.chain.unfiltered > all.chain
    rm -r chain
    chainSplit chain all.chain
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain/chain
    hgLoadChain hg17 chr1_chainTetNig1Filt5k chr1.chain
# featureBits -chrom=chr1 hg17 refGene:cds chainTetNig1Filt5kLink -enrichment
# refGene:cds 1.246%, chainTetNig1Filt5kLink 1.487%, both 0.789%, cover 63.33%,
# enrich 42.58x
# this cleans it up a lot with little reduction in coverage.
# check in browser - filtered version looks good.

# add all chains for minScore=5000 filtered chains
# remove test chain tables for chr1
    ssh hgwdev
    hgsql -e "drop table chr1_chainTetnig1;" hg17
    hgsql -e "drop table chr1_chainTetnig1Link;" hg17
    hgsql -e "drop table chr1_chainTetNig1Filt5k;" hg17
    hgsql -e "drop table chr1_chainTetNig1Filt5kLink;" hg17

    cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain/chain
    foreach i (*.chain)
        set c = $i:r
        hgLoadChain hg17 ${c}_chainTetNig1 $i
        echo done $c
    end

# NET TETRAODON (tetNig1) BLASTZ (DONE, 2004-08-26, hartera)
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
    mkdir preNet
    cd chain
    foreach i (*.chain)
       echo preNetting $i
       /cluster/bin/i386/chainPreNet $i ../../S1.len ../../S2.len \
                                     ../preNet/$i
    end
    cd ..
    mkdir n1
    cd preNet
    foreach i (*.chain)
      set n = $i:r.net
      echo primary netting $i
      /cluster/bin/i386/chainNet $i -minSpace=1 ../../S1.len ../../S2.len \
                                 ../n1/$n /dev/null
    end
    cd ..
    cat n1/*.net | /cluster/bin/i386/netSyntenic stdin noClass.net
    # memory usage 55373824, utime 415 s/100, stime 45
    # Add classification info using db tables:
    cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
    # netClass looks for ancient repeats in one of the databases
    # hg17 has this table - hand-curated by Arian but this is for
    # human-rodent comparisons so do not use here, use -noAr option
    mkdir -p /cluster/bluearc/hg17/linSpecRep.notInTetraodon
    mkdir -p /cluster/bluearc/tetNig1/linSpecRep.notInHuman
    cp /iscratch/i/hg17/linSpecRep.notInTetraodon/* \
       /cluster/bluearc/hg17/linSpecRep.notInTetraodon
    cp /iscratch/i/tetNig1/linSpecRep.notInHuman/* \
       /cluster/bluearc/tetNig1/linSpecRep.notInHuman

    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
    time netClass noClass.net hg17 tetNig1 tetNig1.net \
          -tNewR=/cluster/bluearc/hg17/linSpecRep.notInTetraodon \
          -qNewR=/cluster/bluearc/tetNig1/linSpecRep.notInHuman -noAr
    # 54.100u 31.890s 2:20.01 61.4%   0+0k 0+0io 197pf+0w
    netFilter -minGap=10 tetNig1.net |  hgLoadNet hg17 netTetNig1 stdin
    # featureBits hg17 refGene:cds netTetNig1 -enrichment
    # refGene:cds 0.978%, netTetNig1 25.095%, both 0.778%, cover 79.53%, 
    # enrich 3.17x

# TWINSCAN 1.3 GENE PREDICTIONS (Done, 2004-Aug-26, heather)

    cd /cluster/data/hg17/bed
    mkdir twinscan
    tarFile=hg17_TS13_pseudomasked.tar.gz
    wget http://genes.cs.wustl.edu/predictions/human/NCBI35/hg17_TS13_pseudomasked.tar.gz
    wget http://genes.cs.wustl.edu/predictions/human/NCBI35/md5sum.txt

    # check file transferred correctly
    grep gz md5sum.txt > gz.sum
    md5sum $tarFile | diff - gz.sum

    # extract
    tar xvfz $tarFile
    unset tarFile 
    
    # check that files unzipped and untarred correctly
    # expect no differences
    cd chr_gtf
    grep gtf ../md5sum.txt > md5sum.txt
    cd ../chr_ptx
    grep ptx ../md5sum.txt > md5sum.txt
    cd ../chr_tx
    grep tx ../md5sum.txt > md5sum.txt
    cd ..
    
    md5sum chr_gtf/* > gtf.sum
    diff gtf.sum chr_gtf/md5sum.txt

    md5sum chr_ptx/* > ptx.sum
    diff ptx.sum chr_ptx/md5sum.txt

    md5sum chr_tx/* > tx.sum
    diff tx.sum chr_tx/md5sum.txt

    # pare down protein FASTA header to id and add missing .a:
    foreach c (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y)
      echo chr$c
      perl -wpe 's/^(\>\S+)\s.*$/$1.a/' < chr_ptx/chr$c.ptx > chr_ptx/chr$c-fixed.fa
    end
    ldHgGene hg17 twinscan chr_gtf/chr*.gtf -gtf -genePredExt
    hgPepPred hg17 generic twinscanPep chr_ptx/chr*-fixed.fa

# MAKE VSTETNIG1 DOWNLOADABLES (DONE, 2004-09-08, hartera)
#    Replace with gzipped versions (DONE 2004-09-14 kate)
    ssh kksilo
    # zip chains and nets
    cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
    cp all.chain tetNig1.chain
    zip -j /cluster/data/hg17/zip/tetNig1.chain.zip tetNig1.chain
    rm tetNig1.chain
    zip -j /cluster/data/hg17/zip/tetNig1.net.zip tetNig1.net
   
    ssh hgwdev
    # copy chains and nets to downloads area
    set gp = /usr/local/apache/htdocs/goldenPath/hg17
    mkdir -p $gp/vsTetNig1
    cd $gp/vsTetNig1
    mv /cluster/data/hg17/zip/tetNig1*.zip .
    md5sum *.zip > md5sum.txt

    # move axt files to downloads area and zip
    cd /cluster/data/hg17/bed/blastz.tetNig1/axtChrom
    mkdir -p $gp/vsTetNig1/axtChrom
    cp -p *.axt $gp/vsTetNig1/axtChrom
    cd $gp/vsTetNig1/axtChrom
    gzip *.axt
    md5sum *.gz > md5sum.txt

    # Copy over & edit README.txt w/pointers to chain, net formats.

# BLASTZ TETRAODON (tetNig1) CLEANUP (DONE, 2004-09-10, hartera)
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.tetNig1
    nice rm -rf raw &
    nice rm -rf lav &
    nice rm axtChain/run1/chain/* &
    nice gzip {axt,psl}Chrom/* axtChain/{all.chain,*.net} &


# regulatory potential 2X track (WORKING - 2004-09-14 - Hiram)
    ssh eieio
    mkdir /cluster/store3/gs.18/build35/bed/regPotential2X
    mkdir /cluster/store3/gs.18/build35/bed/regPotential3X
    cd /cluster/data/hg17/bed
    ln -s /cluster/store3/gs.18/build35/bed/regPotential2X .
    ln -s /cluster/store3/gs.18/build35/bed/regPotential3X .
    cd regPotential2X
    wget --timestamping 'http://www.bx.psu.edu/~james/stuff/rp_kit.tgz' .
    tar xvzf rp_kit.tgz
    # fixup the hmr_rp_score.sh and hm_rp_score.sh to set
    #	RP=. to read: RP=/cluster/data/hg17/bed/regPotential2X/rp_kit
    #	And fix the usage of SHIFT and WINDOW, the following diff shows
    #	the changes:
    #	5c5
    #	< RP_DIR=/cluster/data/hg17/bed/regPotential2X/rp_kit
    #	---
    #	> RP_DIR=.
    #	8,9c8,9
    #	< MAPPING=rp_kit/hm_5a_mapping.txt
    #	< MATRIX=rp_kit/hm_5a+3_scoreMatrix.dat
    #	---
    #	> MAPPING=hm_5a_mapping.txt
    #	> MATRIX=hm_5a+3_scoreMatrix.dat
    #	12c12
    #	< SHIFT=1
    #	---
    #	> SHIFT=5
    #	24,25c24,25
    #	<     --shiftAmount $SHIFT \
    #	<     --windowSize $WINDOW \
    #	---
    #	>     --shiftAmount 5 \
    #	>     --windowSize 100 \

    mkdir maf
    for A in  `(cd /cluster/data/hg17/bed/blastz.mm5/axtNet; ls chr*.axt)`
    do
	C=${A/.axt}
	echo "/cluster/data/hg17/bed/blastz.mm5/axtNet/${A} -> maf/${C}.maf.gz"
	axtToMaf /cluster/data/hg17/bed/blastz.mm5/axtNet/${A} \
		/cluster/data/hg17/chrom.sizes /cluster/data/mm5/chrom.sizes \
		stdout | gzip > maf/${C}.maf.gz
    done
    # Replace bad chr5 axtNet and mafNet (2006-01-05 kate) 

    #	a valid java runtime is only on hgwdev.  This is a java procedure
    ssh hgwdev
    cd /cluster/data/hg17/bed/regPotential2X
    mkdir rp_scores
    #  WARNING - the following loop takes almost 12 hours !
    for M in maf/chr*.maf.gz
    do
	C=${M/.maf.gz}
	C=${C#maf/}
	echo "$M -> rp_scores/$C.score.gz"
	(zcat ${M} | ./rp_kit/hm_rp_score.sh /dev/stdin /dev/stderr 2>&1 >/dev/null) | sort -n | \
	    gzip > rp_scores/${C}.score.gz
    done
    #	real    709m55.805s
    #	user    754m51.030s
    #	sys     20m11.000s

    #	Back to the file server to create the wiggle data

    ssh eieio
    cd /cluster/data/hg17/bed/regPotential2X
    mkdir wigData dataLimits
    for S in rp_scores/chr*.score.gz
    do
	C=${S/.score.gz}
	C=${C#rp_scores/}
	echo "$S -> wigData/$C.wig"
	zcat $S | sort -n | \
	    wigAsciiToBinary -chrom=$C -dataSpan=1 \
	    -wibFile=wigData/$C stdin 2> dataLimits/$C.limits
    done
    #	real    313m0.567s
    #	user    285m37.319s
    #	sys     23m8.301s

    #	Loading the table on hgwdev
    ssh hgwdev
    cd /cluster/data/hg17/bed/regPotential2X/wigData
    mkdir /gbdb/hg17/wib/regPotential2X
    ln -s `pwd`/*.wib /gbdb/hg17/wib/regPotential2X
    time hgLoadWiggle -pathPrefix=/gbdb/hg17/wib/regPotential2X \
	hg17 regPotential2X chr*.wig
    #	real    2m29.668s
    #	user    0m33.380s
    #	sys     0m8.200s

# regulatory potential 3X track (WORKING - 2004-09-14 - Hiram)
    #	Expects groundwork done above in the 2X track
    #	a valid java runtime is only on hgwdev.  This is a java procedure
    ssh hgwdev
    cd /cluster/data/hg17/bed/regPotential3X
    ln -s ../regPotential2X/rp_kit/hmr_rp_score.sh .
    mkdir rp_scores
    #  WARNING - the following loop takes almost 12 hours !
    for M in maf/chr*.maf.gz
    do
	C=${M/.maf.gz}
	C=${C#maf/}
	echo "$M -> rp_scores/$C.score.gz"
	(zcat ${M} | ./rp_kit/hm_rp_score.sh /dev/stdin /dev/stderr 2>&1 >/dev/null) | sort -n | \
	    gzip > rp_scores/${C}.score.gz
    done
    #	real    613m8.230s
    #	user    623m7.110s
    #	sys     20m24.550s

    #	Back to the file server to create the wiggle data

    ssh eieio
    cd /cluster/data/hg17/bed/regPotential3X
    mkdir wigData dataLimits

    for S in rp_scores/chr*.score.gz
    do
	C=${S/.score.gz}
	C=${C#rp_scores/}
	echo "$S -> wigData/$C.wig"
	zcat $S | sort -n | \
	    wigAsciiToBinary -chrom=$C -dataSpan=1 \
		-wibFile=wigData/$C stdin 2> dataLimits/$C.limits
    done

    #	Loading the table on hgwdev
    ssh hgwdev
    cd /cluster/data/hg17/bed/regPotential3X/wigData
    mkdir /gbdb/hg17/wib/regPotential3X
    ln -s `pwd`/*.wib /gbdb/hg17/wib/regPotential3X
    time hgLoadWiggle -pathPrefix=/gbdb/hg17/wib/regPotential3X \
	hg17 regPotential3X chr*.wig
    #	real    1m45.568s
    #	user    0m32.740s
    #	sys     0m6.140s

# regulatory potential 5X track (DONE - 2005-09-19 - Daryl)
    ssh kkstore02
    mkdir -p /cluster/data/hg17/bed/regPotential5X/rp_scores
    cd /cluster/data/hg17/bed/regPotential5X/rp_scores
    wget -r -l 1 -nH http://www.bx.psu.edu/~james/rp/hg17panTro1mm5rn3canFam1/all_truncate.tar
    tar xvf all_truncate.tar
    cd /cluster/data/hg17/bed/regPotential5X
    mkdir -p wigData dataLimits 
    cd wigData
    ## 8 minutes
    for S in ../rp_scores/chr*.scores.truncated.gz
    do
	C=${S/.scores.truncated.gz}
	C=${C#../rp_scores/}
	echo "$S -> wigData/$C.wig"
	zcat $S | wigEncode stdin $C.wig $C.wib 2> ../dataLimits/$C.limits
    done

    #	Loading the table on hgwdev
    ssh hgwdev
    cd /cluster/data/hg17/bed/regPotential5X/wigData
    mkdir -p /gbdb/hg17/wib/regPotential5X
    chmod o+rx /gbdb/hg17/wib/regPotential5X
    ln -s /cluster/data/hg17/bed/regPotential5X/wigData/*.wib /gbdb/hg17/wib/regPotential5X
    chmod o+r /gbdb/hg17/wib/regPotential5X/ch*wib
    time hgLoadWiggle -pathPrefix=/gbdb/hg17/wib/regPotential5X hg17 regPotential5X chr*.wig
    # 57.720u 9.960s 2:26.05 46.3%    0+0k 0+0io 213pf+0w

# SGP GENES (DONE 9/17/04 angie)
    ssh eieio
    mkdir /cluster/data/hg17/bed/sgp
    cd /cluster/data/hg17/bed/sgp
    foreach chr (`awk '{print $1;}' ../../chrom.sizes`)
      wget http://genome.imim.es/genepredictions/H.sapiens/golden_path_200405/SGP/$chr.gtf
      wget http://genome.imim.es/genepredictions/H.sapiens/golden_path_200405/SGP/$chr.prot
    end
    # Add ".1" suffix to each item in .prot's, to match transcript_id's in gtf
    cp /dev/null sgpPep.fa
    foreach f (chr*.prot)
      perl -wpe 's/^(>chr\S+)/$1.1/' $f >> sgpPep.fa
    end
    ssh hgwdev
    cd /cluster/data/hg17/bed/sgp
    ldHgGene -gtf -genePredExt hg17 sgpGene chr*.gtf
    hgPepPred hg17 generic sgpPep sgpPep.fa

# SGP GENES (UPDATE 1/18/2006)
    sgpPep table dropped, replaced by hgc generated protein seq in browser

# LIFTOVER RNAGENE FROM HG16 (09/29/04, acs)
    cd /cluster/data/hg17/bed
    mkdir rnaGene
    cd rnaGene
    liftOver -gff /cluster/data/hg16/bed/rnaGene/all.gff \
	/cluster/data/hg16/bed/liftOver/hg16ToHg17.over.chain \
	rnaGeneLift.gff rnaGeneMiss.gff
    # 7204 records passed, 16 failed
    hgsql hg17 < ~/kent/src/hg/lib/rnaGene.sql
    hgRnaGenes hg17 rnaGeneLift.gff

    
# BUILD BioCyc TABLES (DONE 10/1/04 Fan)

  - Create bioCycMapDesc table.

	CREATE TABLE bioCycMapDesc (
  	mapID varchar(40) NOT NULL default '',
  	description varchar(255) NOT NULL default '',
  	KEY mapID (mapID)
	) TYPE=MyISAM;

  - Crate bioCycPathway table.

	CREATE TABLE bioCycPathway (
  	kgID varchar(40) NOT NULL default '',
  	geneID varchar(40) NOT NULL default '',
  	mapID varchar(40) NOT NULL default '',
  	KEY kgID (kgID),
 	KEY geneID (geneID),
  	KEY mapID (mapID)
	) TYPE=MyISAM;

  Using data files sent by Peter Carp from SRI,
  per Peter's email of 10/1/04, they don't have recent update,
  so data files received last year are used.
  
  Save the BioCyc Pathway name and description table as names.txt.
  Save the pathway data file as gene-pathway.dat.
  Make sure there is no extra ^M at end of the lines.

  hgsql hg17 -e	'LOAD DATA local INFILE 'names.txt' into table bioCycMapDesc'

  Run hgBioCyc program to generate the file bioCycPathway.tab.

	 hgBioCyc gene-pathway.dat hg17

  Load into hg17.
 
	 hgsql hg17 -e 'LOAD DATA local INFILE "bioCycPathway.tab" into table bioCycPathway'
	
# MAKING FOLDUTR TABLES (DONE - 2004-10-4 Fan)
# First set up directory structure and extract UTR sequence on hgwdev
    ssh hgwdev
    mkdir -p /cluster/data/hg17/bed/rnaStruct
    cd /cluster/data/hg17/bed/rnaStruct
    mkdir -p utr3/split utr5/split utr3/fold utr5/fold
    utrFa hg17 knownGene utr3 utr3/utr.fa
    utrFa hg17 knownGene utr5 utr5/utr.fa

# Split up files and make files that define job.
    ssh kk
    cd /cluster/data/hg17/bed/rnaStruct
    faSplit sequence utr3/utr.fa 50000 utr3/split/s
    faSplit sequence utr5/utr.fa 50000 utr5/split/s
    ls -1 utr3/split > utr3/in.lst
    ls -1 utr5/split > utr5/in.lst
    cd utr3
    cat > gsub <<end
#LOOP
rnaFoldBig split/\$(path1) fold
#ENDLOOP
end
    cp gsub ../utr5

# Do cluster run for 3' UTRs
    gensub2 in.lst single gsub spec
    para create spec
    para try
    para push
# Completed: 37244 of 37244 jobs
# CPU time in finished jobs:    1036479s   17274.64m   287.91h   12.00d  0.033 y
# IO & Wait Time:                112286s    1871.44m    31.19h    1.30d  0.004 y
# Average job time:                  31s       0.51m     0.01h    0.00d
# Longest job:                     3370s      56.17m     0.94h    0.04d
# Submission to last job:          4355s      72.58m     1.21h    0.05d

# Do cluster run for 5' UTRs 
    cd ../utr5
    gensub2 in.lst single gsub spec
    para create spec
    para try
    para push
# Completed: 29817 of 29817 jobs
# CPU time in finished jobs:      98143s    1635.72m    27.26h    1.14d  0.003 y
# IO & Wait Time:                105763s    1762.71m    29.38h    1.22d  0.003 y
# Average job time:                   7s       0.11m     0.00h    0.00d
# Longest job:                     2133s      35.55m     0.59h    0.02d
# Submission to last job:          2465s      41.08m     0.68h    0.03d

# Load database
    ssh hgwdev
    cd /cluster/data/hg17/bed/rnaStruct/utr5
    hgLoadRnaFold hg17 foldUtr5 fold
    cd ../utr3
    hgLoadRnaFold hg17 foldUtr3 fold

# Clean up
    rm -r split fold err batch.bak
    cd ../utr5
    rm -r split fold err batch.bak

####### BUILD RGD HUMAN QTL TRACKS (DONE 10/7/04 Fan) ##############

mkdir -p /cluster/store8/rgd/human041007
ln -s /cluster/store8/rgd/human041007 /cluster/data/hg17/bed/rgdQtl
cd /cluster/data/hg17/bed/rgdQtl

# download data files from RGD

wget --timestamp ftp://rgd.mcw.edu/pub/RGD_genome_annotations/human/human_QTL.gff

# remove extra line feed character at the end of lines

rmLf human_QTL.gff > rgdQtl.gff

# create rgdQtl.tab
awk '{print $1"\t"$4"\t"$5"\t"$10}'  rgdQtl.gff |sed -e 's/Chr/chr/g'| \
sed -e 's/"//g' |sed -e 's/RGD://g' | sed -e 's/;//g' > rgdQtl.tab

# create rgdQtlLink.tab

awk '{printf "%s\t%s\t", $12, $10; for (i = 14;i <= NF; ++i ) {printf "%s ", $i} printf "\n"} ' rgdQtl.gff | \
sed -e 's/"//g'| sed -e 's/RGD://g' | sed -e 's/;//g'| sed -e 's/Note//g' > rgdQtlLink.tab

# load rgdQtl table
hgLoadBed hg17 rgdQtl rgdQtl.tab

# check rgdQtl table
checkTableCoords hg17 rgdQtl

# load rgdQtlLink table
hgsql hg17 -e "drop table hg17.rgdQtlLink;"
hgsql hg17 <~/kent/src/hg/lib/rgdQtlLink.sql
hgsql hg17 -e 'load data local infile "rgdQtlLink.tab" into table hg17.rgdQtlLink;'

# updated trackDb.ra under /kent/src/hg/makeDb/trackDb/human/hg17 and
# added rgdQtl.html.

#### AFFYMETRIX HG-U133 PLUS TRACK (DONE, 2004-10-11, hartera)
     ssh hgwdev
     mkdir -p /projects/compbio/data/microarray/affyHuman/HG-U133Plus2
     # Go to 
#http://www.affymetrix.com/support/technical/byproduct.affx?product=hg-u133-plus 
     # and download the consensus and exemplar sequences to this directory
     cd /projects/compbio/data/microarray/affyHuman/HG-U133Plus2
     unzip HG-U133_Plus_2_consensus.zip
     unzip HG-U133_Plus_2_exemplar.zip
     cat HG-U133_Plus_2_consensus HG-U133_Plus_2_exemplar >> U133Plus2_all.fa
     perl -pi.bak -e "s/(consensus|exemplar):HG-U133_Plus_2:/U133+2:/" \
                     U133Plus2_all.fa
     # remove ";" from probe set names
     perl -pi.bak -e "s/;//" U133Plus2_all.fa
     # clean up
     rm *.zip *.bak
     mkdir -p /cluster/data/hg17/bed/affyU133Plus2.2004-10-11
     cp U133Plus2_all.fa /cluster/data/hg17/bed/affyU133Plus2.2004-10-11
     # Set up cluster job to align consensus/exemplars to hg16
     ssh kkr1u00
     mkdir -p /iscratch/i/affy
     mv /cluster/data/hg17/bed/affyU133Plus2.2004-10-11/U133Plus2_all.fa \
        /iscratch/i/affy
     iSync

     ssh kk
     cd /cluster/data/hg17/bed/affyU133Plus2.2004-10-11
     ls -1 /iscratch/i/affy/U133Plus2_all.fa > affy.lst
     ls -1 /iscratch/i/gs.18/build35/maskedContigs/* > allctg.lst

    cat << '_EOF_' > template.sub
#LOOP
/cluster/bin/i386/blat -fine -mask=lower -minIdentity=95 -ooc=/iscratch/i/gs.18/build35/hg17.11.ooc $(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'
    # << for emacs
    gensub2 allctg.lst affy.lst template.sub para.spec
    mkdir psl
    para create para.spec
    para try, para check, para push .....
# para time
# Completed: 380 of 380 jobs
# CPU time in finished jobs:      24533s     408.88m     6.81h    0.28d  0.001 y
# IO & Wait Time:                  2180s      36.34m     0.61h    0.03d  0.000 y
# Average job time:                  70s       1.17m     0.02h    0.00d
# Longest job:                      751s      12.52m     0.21h    0.01d
# Submission to last job:          2425s      40.42m     0.67h    0.03d

    # Do sort, best in genome filter, and convert to chromosome coordinates
    # to create affyU133Plus2.psl
    pslSort dirs raw.psl tmp psl
                                                                                
    # use filter parameters for these sequences. only use alignments that
    # cover 30% of sequence and have at least 95% identity in aligned region.
    # minAli = 0.97 too high. low minCover as a lot of n's in these sequences
  pslReps -minCover=0.3 -minAli=0.95 -nearTop=0.005 raw.psl contig.psl /dev/null
    liftUp affyU133Plus2.psl ../../jkStuff/liftAll.lft warn contig.psl
    perl -pi.bak -e "s/U133\+2://" affyU133Plus2.psl
    # load into the database
    ssh hgwdev
    cd /cluster/data/hg17/bed/affyU133Plus2.2004-10-11
    hgLoadPsl hg17 affyU133Plus2.psl
    # Add sequence data to database
        # Copy probe sequence to /gbdb if it isn't already
    mkdir -p /gbdb/hgFixed/affyProbes
    cd /gbdb/hgFixed/affyProbes
    ln -s /projects/compbio/data/microarray/affyHuman/HG-U133Plus2/U133Plus2_all.fa .
    cd /cluster/data/hg17/bed/affyU133Plus2.2004-10-11
    hgLoadSeq -abbr=U133+2: hg17 /gbdb/hgFixed/affyProbes/U133Plus2_all.fa
    
    # clean up
    rm -r psl tmp err contig.psl raw.psl *.bak psl.tab seq.tab
# Added knownToU133Plus2 track (2004-10-14) - see GeneSorter section


#### MAF COVERAGE FIGURES FOR ADAM (DONE 10/18/04 angie)
    # First, get ranges of target coverage:
    ssh eieio
    mkdir /cluster/data/hg17/bed/var_multiz.2004-08-12/coverage
    cd /cluster/data/hg17/bed/var_multiz.2004-08-12/coverage/
    cat /cluster/data/hg17/bed/var_multiz.2004-08-12/maf.09-12-04/*.maf \
    | nice mafRanges -notAllOGap stdin hg17 hg17.mafRanges.bed
    # Get pairwise coverage as well.
    ssh kolossus
    cd /cluster/data/hg17/bed/var_multiz.2004-08-12/coverage/
    cat /cluster/bluearc/hg17/multiz8way/rn3/*.maf \
    | nice mafRanges -notAllOGap stdin hg17 hg17.rn3.mafRanges.bed
    cat /cluster/bluearc/hg17/multiz8way/mm5/*.maf \
    | nice mafRanges -notAllOGap stdin hg17 hg17.mm5.mafRanges.bed
    cat /cluster/bluearc/hg17/multiz8way/galGal2/*.maf \
    | nice mafRanges -notAllOGap stdin hg17 hg17.galGal2.mafRanges.bed
    cat /cluster/bluearc/hg17/multiz8way/fr1/*.maf \
    | nice mafRanges -notAllOGap stdin hg17 hg17.fr1.mafRanges.bed

    ssh hgwdev
    cd /cluster/data/hg17/bed/var_multiz.2004-08-12/coverage
    # To make subsequent intersections a bit quicker, output a bed with 
    # duplicate/overlapping ranges collapsed:
    nice featureBits hg17 hg17.mafRanges.bed \
      -bed=hg17.mafRangesCollapsed.bed
#1147548420 bases of 2866216770 (40.037%) in intersection
    foreach other (mm5 rn3 galGal2 fr1)
      nice featureBits hg17 hg17.$other.mafRanges.bed \
      -bed=hg17.${other}.mafRangesCollapsed.bed
    end
#1013348528 bases of 2866216770 (35.355%) in intersection
#975533772 bases of 2866216770 (34.036%) in intersection
#101623034 bases of 2866216770 (3.546%) in intersection
#46737824 bases of 2866216770 (1.631%) in intersection

    # mafCoverage barfs currently, so pass on this for now:
    #cat ../maf.09-12-04/*.maf \
    #| nice mafCoverage -count=2 hg17 stdin > hg17.mafCoverage

    # Intersect maf target coverage with gene regions -- 
    # use Adam's knownGene region files:
    nice featureBits hg17 -enrichment \
      ../phastCons/stats2/knownGenesCds.bed \
      hg17.mafRangesCollapsed.bed \
      -bed=hg17.mafCds.bed
#knownGenesCds.bed 1.166%, hg17.mafRangesCollapsed.bed 40.037%, both 1.111%, cover 95.36%, enrich 2.38x

    nice featureBits hg17 -enrichment \
      ../phastCons/stats2/knownGenesUtr3.bed \
      hg17.mafRangesCollapsed.bed \
      -bed=hg17.mafUtr3.bed
#knownGenesUtr3.bed 0.918%, hg17.mafRangesCollapsed.bed 40.037%, both 0.662%, cover 72.18%, enrich 1.80x
    nice featureBits hg17 -enrichment \
      ../phastCons/stats2/knownGenesUtr5.bed \
      hg17.mafRangesCollapsed.bed \
      -bed=hg17.mafUtr5.bed
#knownGenesUtr5.bed 0.266%, hg17.mafRangesCollapsed.bed 40.037%, both 0.198%, cover 74.42%, enrich 1.86x

    # Intersect pairwise target coverages with gene regions:
    foreach other (mm5 rn3 galGal2 fr1)
      nice featureBits hg17 -enrichment \
        ../phastCons/stats2/knownGenesCds.bed \
        hg17.$other.mafRangesCollapsed.bed -bed=hg17.${other}Cds.bed
      nice featureBits hg17 -enrichment \
        ../phastCons/stats2/knownGenesUtr3.bed \
        hg17.$other.mafRangesCollapsed.bed -bed=hg17.${other}Utr3.bed
      nice featureBits hg17 -enrichment \
        ../phastCons/stats2/knownGenesUtr5.bed \
        hg17.$other.mafRangesCollapsed.bed -bed=hg17.${other}Utr5.bed
    end
#knownGenesCds.bed 1.166%, hg17.mm5.mafRangesCollapsed.bed 35.355%, both 1.093%, cover 93.74%, enrich 2.65x
#knownGenesUtr3.bed 0.918%, hg17.mm5.mafRangesCollapsed.bed 35.355%, both 0.618%, cover 67.37%, enrich 1.91x
#knownGenesUtr5.bed 0.266%, hg17.mm5.mafRangesCollapsed.bed 35.355%, both 0.186%, cover 69.81%, enrich 1.97x
#knownGenesCds.bed 1.166%, hg17.rn3.mafRangesCollapsed.bed 34.036%, both 1.071%, cover 91.85%, enrich 2.70x
#knownGenesUtr3.bed 0.918%, hg17.rn3.mafRangesCollapsed.bed 34.036%, both 0.597%, cover 65.09%, enrich 1.91x
#knownGenesUtr5.bed 0.266%, hg17.rn3.mafRangesCollapsed.bed 34.036%, both 0.179%, cover 67.33%, enrich 1.98x
#knownGenesCds.bed 1.166%, hg17.galGal2.mafRangesCollapsed.bed 3.546%, both 0.779%, cover 66.84%, enrich 18.85x
#knownGenesUtr3.bed 0.918%, hg17.galGal2.mafRangesCollapsed.bed 3.546%, both 0.194%, cover 21.12%, enrich 5.96x
#knownGenesUtr5.bed 0.266%, hg17.galGal2.mafRangesCollapsed.bed 3.546%, both 0.056%, cover 21.03%, enrich 5.93x
#knownGenesCds.bed 1.166%, hg17.fr1.mafRangesCollapsed.bed 1.631%, both 0.714%, cover 61.26%, enrich 37.57x
#knownGenesUtr3.bed 0.918%, hg17.fr1.mafRangesCollapsed.bed 1.631%, both 0.073%, cover 7.92%, enrich 4.86x
#knownGenesUtr5.bed 0.266%, hg17.fr1.mafRangesCollapsed.bed 1.631%, both 0.039%, cover 14.82%, enrich 9.09x


# ALTERNATIVE CPG ISLANDS (DONE 10/14/04 angie)
    ssh eieio
    nice tcsh
    mkdir /cluster/data/hg17/bed/cpgIslandAlt
    cd /cluster/data/hg17/bed/cpgIslandAlt
    # Try cpg_ash (WUSTL program modified to not chop islands in half before 
    # scoring) with default params:
    cp /dev/null cpg_ash.default.cpg
    foreach f (../../?{,?}/chr*.fa.masked)
      echo running on $f:t:r:r
      ~angie/cb/hg3rdParty/cpgIslands/cpg_ash.exe $f >> cpg_ash.default.cpg
    end
    awk -f ../cpgIsland/filter.awk cpg_ash.default.cpg > cpgIslandAlt.bed
    # Run Andy Law's script on masked seq:
    cp /dev/null cpgIslandGgfAndyMasked.bed
    foreach f (../../?{,?}/chr*.fa.masked)
      set chr = $f:t:r:r
      echo running on $chr
      /cluster/home/angie/bin/$MACHTYPE/preProcGgfAndy $f \
      | /cluster/home/angie/ggf-andy-cpg-island.pl \
      | perl -wpe 'chomp; ($s,$e,$cpg,$n,$c,$g,$oE) = split("\t"); $s--; \
                   $gc = $c + $g;  $pCpG = (100.0 * 2 * $cpg / $n); \
                   $pGc = (100.0 * $gc / $n); \
                   $_ = "'$chr'\t$s\t$e\tCpG: $cpg\t$n\t$cpg\t$gc\t" . \
                        "$pCpG\t$pGc\t$oE\n";' \
      >> cpgIslandGgfAndyMasked.bed
    end
    # Compare enrichment for knownGene upstream -- an uphill battle for 
    # programs closer to meeting the stated length, GC, O/E params!
    ssh hgwdev
    nice featureBits hg17 -enrichment knownGene:upstream:1000 \
      /cluster/data/hg17/bed/cpgIsland/cpgIsland.bed
#knownGene:upstream:1000 0.857%, cpgIsland.bed 0.741%, both 0.166%, cover 19.37%, enrich 26.13x
    nice featureBits hg17 -enrichment knownGene:upstream:1000 \
      /cluster/data/hg17/bed/cpgIslandAlt/cpgIslandAlt.bed
#knownGene:upstream:1000 0.857%, cpgIslandAlt.bed 1.075%, both 0.200%, cover 23.38%, enrich 21.76x
    nice featureBits hg17 -enrichment knownGene:upstream:1000 \
      /cluster/data/hg17/bed/cpgIslandAlt/cpgIslandGgfAndyMasked.bed
#knownGene:upstream:1000 0.857%, cpgIslandGgfAndyMasked.bed 1.964%, both 0.292%, cover 34.06%, enrich 17.34x
    cd /cluster/data/hg17/bed/cpgIslandAlt
    sed -e 's/cpgIslandExt/cpgIslandAlt/g' \
      ~/kent/src/hg/lib/cpgIslandExt.sql > cpgIslandAlt.sql
    hgLoadBed -noBin -tab -sqlTable=cpgIslandAlt.sql \
      hg17 cpgIslandAlt cpgIslandAlt.bed
#Loaded 29998 elements of size 10
    sed -e 's/cpgIslandExt/cpgIslandGgfAndyMasked/g' \
      ~/kent/src/hg/lib/cpgIslandExt.sql > cpgIslandGgfAndyMasked.sql
    hgLoadBed -noBin -tab -sqlTable=cpgIslandGgfAndyMasked.sql \
      hg17 cpgIslandGgfAndyMasked cpgIslandGgfAndyMasked.bed
#Loaded 80555 elements of size 10
    # Quick length stats:
    hgsql hg17 -e 'select min(length), avg(length), max(length) from cpgIslandExt'
#|         201 |    764.1913 |       40058 |
    hgsql hg17 -e 'select min(length), avg(length), max(length) from cpgIslandAlt'
#|         200 |   1026.9194 |       32440 |
    hgsql hg17 -e 'select min(length), avg(length), max(length) from cpgIslandGgfAndyMasked'
#|         200 |    698.8257 |      100308 |
    # 1/26/05: Make better island names in cpgIslandGgfAndyMasked,
    # for Dave Burt's cross-species island comparisons.
    ssh eieio
    cd /cluster/data/hg17/bed/cpgIslandAlt
    mv cpgIslandGgfAndyMasked.bed cpgIslandGgfAndyMasked.bed.orig
    perl -wpe '@w=split("\t"); $w[3] = "hg17.$w[0]." . ($w[1]+1) . ".$w[2]"; \
               $_ = join("\t", @w);' \
      cpgIslandGgfAndyMasked.bed.orig \
    > cpgIslandGgfAndyMasked.bed
    # Now liftOver islands from mm5, rn3, galGal2:
    ssh kolossus
    cd /cluster/data/hg17/bed/cpgIslandAlt
    foreach match (50 95)
      liftOver /cluster/data/mm5/bed/cpgIslandGgfAndy/cpgIslandGgfAndyMasked.bed \
        /cluster/data/mm5/bed/bedOver/mm5Tohg17.chain -minMatch=0.$match \
        cpgIslandGAMFromMm5_$match.bed cpgIslandGAMFromMm5_$match.unmapped
      liftOver /cluster/data/rn3/bed/cpgIslandGgfAndy/cpgIslandGgfAndyMasked.bed \
        /cluster/data/rn3/bed/bedOver/rn3ToHg17.over.chain -minMatch=0.$match \
        cpgIslandGAMFromRn3_$match.bed cpgIslandGAMFromRn3_$match.unmapped
      liftOver /cluster/data/galGal2/bed/cpgIslandGgfAndy/cpgIslandGgfAndyMasked.bed \
        /cluster/data/galGal2/bed/bedOver/galGal2ToHg17.over.chain -minMatch=0.$match \
        cpgIslandGAMFromGalGal2_$match.bed cpgIslandGAMFromGalGal2_$match.unmapped
    end
    # Load up the renamed islands as well as
    ssh hgwdev
    cd /cluster/data/hg17/bed/cpgIslandAlt
    hgLoadBed -noBin -tab -sqlTable=cpgIslandGgfAndyMasked.sql \
      hg17 cpgIslandGgfAndyMasked cpgIslandGgfAndyMasked.bed


# MAKE UNIGENE/SAGE TRACK (DONE - 2004-10-15 Fan)

# First get SAGE data and determine which version of UniGene to use first
    ssh hgwdev
    cd ~/kent/src/hg/sage
    make
    # XXX = uniGene build for which SAGE was built -- not necessarily current!
    # Figure out the build number by peeking at this file:
    wget -O - ftp://ftp.ncbi.nih.gov/pub/sage/map/info.txt 2> /dev/null

# UniGene Build #44  Arabidopsis thaliana
# UniGene Build #61  Bos taurus
# UniGene Build #16  Caenorhabditis elegans
# UniGene Build #171  Homo sapiens
# UniGene Build #19  Medicago truncatula
# UniGene Build #138  Mus musculus
# UniGene Build #52  Oryza sativa
# UniGene Build #14  Pinus taeda
# UniGene Build #132  Rattus norvegicus
# UniGene Build #27  Sus scrofa
# UniGene Build #38  Triticum aestivum
# UniGene Build #11  Vitis vinifera
# UniGene Build #41  Zea mays

# From above info, set Version 171 for hg17
    ls /projects/cc/hg/sugnet/uniGene
#   set Version = XXX
    set Version=171
    mkdir /projects/cc/hg/sugnet/sage/sage.$Version
    cd /projects/cc/hg/sugnet/sage/sage.$Version

    wget -r -nH --cut-dirs=2 --timestamp ftp://ftp.ncbi.nih.gov/pub/sage/map/Hs
    wget -r -nH --cut-dirs=2 --timestamp ftp://ftp.ncbi.nih.gov/pub/sage/map/readme.txt
    wget -r -nH --cut-dirs=2 --timestamp ftp://ftp.ncbi.nih.gov/pub/sage/map/info.txt
    wget -r -nH --cut-dirs=2 --timestamp ftp://ftp.ncbi.nih.gov/pub/sage/extr
    wget -r -nH --cut-dirs=2 --timestamp ftp://ftp.ncbi.nih.gov/pub/sage/info

#  That downloaded about 1 GB of data

    cd map/Hs/NlaIII
    unzip -j SAGEmap_tag_ug-rel.zip
    cd ../../../extr/
    ../../scripts/summarizeCounts.pl expCounts.tab ./SAGE_*
    ../../scripts/countGenesPerTag.pl expCounts.tab allTags.count.tab
    ../../scripts/createArraysForTags.pl allTags.count.tab tagExpArrays.tab \
      ./SAGE_*
    ../../scripts/countsPerExp.pl expCounts.tab expList.tab

    cd ../map/Hs/NlaIII/ 
    cat << '_EOF_' > /tmp/t.pl
#!/usr/local/bin/perl

while (<>) {
 chomp($_);
 @p = split(/\t/, $_);
 print "$p[2]\t$p[3]\t$p[0]\n";
}
'_EOF_'
    chmod +x /tmp/t.pl

    cat SAGEmap_tag_ug-rel | /tmp/t.pl | sort | sed -e 's/ /_/g' \
      > SAGEmap_ug_tag-rel_Hs
    cd ../../../extr
    createSageSummary ../map/Hs/NlaIII/SAGEmap_ug_tag-rel_Hs \
      tagExpArrays.tab sageSummary.sage

# Create the uniGene alignments 
# /cluster/data/hg17/uniGene/hg17.uniGene.lifted.pslReps.psl

    # Download of the latest UniGene version is now automated by a 
    # cron job -- see /cluster/home/angie/crontab , 
    # /cluster/home/angie/unigeneVers/unigene.csh .  
    # If hgwdev gets rebooted, that needs to be restarted... maybe there's 
    # a more stable place to set up that cron job.  

    # substitute XXX -> the uniGene version used by SAGE.
    # set Version = XXX
    set Version = 171			(bash: export Version=171)
    cd /projects/cc/hg/sugnet/uniGene/uniGene.$Version
    gunzip Hs.seq.uniq.gz Hs.data.gz
    ../countSeqsInCluster.pl Hs.data counts.tab
    ../parseUnigene.pl Hs.seq.uniq Hs.seq.uniq.simpleHeader.fa leftoverData.tab
    # Distribute UniGene sequence to /iscratch/i/ (kkstore can see /projects)
    ssh kkstore
    set Version = 171 # same as above
    mkdir -p /iscratch/i/uniGene.$Version
    cp -p \
  /projects/cc/hg/sugnet/uniGene/uniGene.$Version/Hs.seq.uniq.simpleHeader.fa \
      /iscratch/i/uniGene.$Version
    ssh kkr1u00
    iSync
    ssh kk
    set Version = 171 # same as above
    mkdir -p /cluster/data/hg17/bed/uniGene.$Version
    cd /cluster/data/hg17/bed/uniGene.$Version
    ls -1S /scratch/hg/gs.18/build35/maskedContigs/*.fa > allctg.lst
    ls -1S /iscratch/i/uniGene.$Version/Hs.seq.uniq.simpleHeader.fa \
      > uniGene.lst
    cat << '_EOF_' > template.sub
#LOOP
/cluster/bin/i386/blat -mask=lower -minIdentity=95 -ooc=/scratch/hg/h/11.ooc $(path1) $(path2)  {check out 
line+ psl/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'

    gensub2 allctg.lst uniGene.lst template.sub para.spec
    para create para.spec
    mkdir psl
    para try
    para check
    para push
# Completed: 380 of 380 jobs
# CPU time in finished jobs:      35994s     599.91m    10.00h    0.42d  0.001 y
# IO & Wait Time:                  1812s      30.19m     0.50h    0.02d  0.000 y
# Average job time:                  99s       1.66m     0.03h    0.00d
# Longest job:                     1497s      24.95m     0.42h    0.02d
# Submission to last job:          1551s      25.85m     0.43h    0.02d

    ssh eieio
    set Version = 171 # same as above
    cd /cluster/data/hg17/bed/uniGene.$Version
    pslSort dirs raw.psl tmp psl >& pslSort.log
    liftUp -type=.psl stdout ../../jkStuff/liftAll.lft warn raw.psl \
    | pslReps -minCover=0.2 -sizeMatters -minAli=0.965 -nearTop=0.002 \
      stdin hg17.uniGene.lifted.pslReps.psl /dev/null
# Processed 141416 alignments

# use hg17.uniGene.lifted.pslReps.psl for building UNIGENE/SAGE track.

    ssh hgwdev
    set Version = 171
    cd /projects/cc/hg/sugnet/sage/sage.$Version/extr
    addAveMedScoreToPsls \
      /cluster/data/hg17/bed/uniGene.$Version/hg17.uniGene.lifted.pslReps.psl \
      sageSummary.sage  uniGene.wscores.bed
    hgLoadBed hg17 uniGene_2 uniGene.wscores.bed
    hgsql hg17 < ~kent/src/hg/lib/sage.sql 
    echo "load data local infile 'sageSummary.sage' into table sage" \
        | hgsql hg17
    cd ../info
    ../../scripts/parseRecords.pl ../extr/expList.tab  > sageExp.tab
    hgsql hg17 < ~/kent/src/hg/lib/sageExp.sql 
    echo "load data local infile 'sageExp.tab' into table sageExp" | hgsql hg17
    # update ~/kent/src/hg/makeDb/trackDb/human/hg17/uniGene_2.html 
    # with current uniGene date.


# CREATE kgSpAlias TABLE FOR PB (Done 10/20/04)

    hgsql hg17 -e \
    'select kgXref.kgID, spID, alias from kgXref, kgAlias where kgXref.kgID=kgAlias.kgID' >j.tmp
    hgsql hg17 -e \
    'select kgXref.kgID, spID, alias from kgXref, kgProtAlias where kgXref.kgID=kgProtAlias.kgID'\
    >>j.tmp
    cat j.tmp|sort -u |grep -v 'kgID' >hg17.kgSpAlias.tab
    rm j.tmp

    hgsql hg17 -e 'drop table kgSpAlias';
    hgsql hg17 < ~/src/hg/lib/kgSpAlias.sql
    hgsql hg17 -e 'load data local infile "hg17.kgSpAlias.tab" into table kgSpAlias'


# SEGMENTAL DUPLICATIONS (DONE 10/21/04 angie)
    ssh hgwdev
    mkdir /cluster/data/hg17/bed/genomicSuperDups
    cd /cluster/data/hg17/bed/genomicSuperDups
    # A tar file containing files for both hg16 and hg17 was downloaded into 
    # /cluster/data/hg16/bed/genomicSuperDups;  move over the hg17 part.
    mv /cluster/data/hg16/bed/genomicSuperDups/bd35 .
    cd bd35
    # A note from Xinwei She about the contents:
#Build35 contains only 2 tables: genomicSuperDups and celeraDupPositive.
    # use tail +2 to skip past the header line:
    # actually, celeraDupPositive.tab.gz has one extra bogus line so +3 for it:
    zcat celeraDupPositive.tab.gz | tail +3 \
    | hgLoadBed -tab -sqlTable=$HOME/kent/src/hg/lib/celeraDupPositive.sql \
        hg17 celeraDupPositive stdin
    zcat genomicSuperDups.tab.gz | tail +2 \
    | hgLoadBed -tab -sqlTable=$HOME/kent/src/hg/lib/genomicSuperDups.sql \
        hg17 genomicSuperDups stdin
    # clean up
    rm bed.tab

# ECGENE TRACK (DONE, 2004-10-29, hartera)
    ssh eieio
    mkdir -p /cluster/data/hg17/bed/ECgene.2004-10-27
    ln -s /cluster/data/hg17/bed/ECgene.2004-10-27 \
          /cluster/data/hg17/bed/ECgene
    cd  /cluster/data/hg17/bed/ECgene
    wget \
"http://genome.ewha.ac.kr/ECgene/download/v1.2_ECgene/v1.2_hg17_low_gene.txt.gz"
    wget \
"http://genome.ewha.ac.kr/ECgene/download/v1.2_ECgene/v1.2_hg17_low_pep.txt.gz"
    gunzip *.gz

    # load database
    ssh hgwdev
    cd /cluster/data/hg17/bed/ECgene
    ldHgGene -predTab hg17 ECgene v1.2_hg17_low_gene.txt
    # 646778 gene predictions
    hgPepPred hg17 tab ECgenePep v1.2_hg17_low_pep.txt
    rm *.tab
    nice gzip *.txt

# LOAD ENSEMBL GENES (DONE, 2004-11-19, hartera)

    mkdir /cluster/data/hg17/bed/ensembl
    cd /cluster/data/hg17/bed/ensembl
    # Get the ensembl protein data from 
    # http://www.ensembl.org/Homo_sapiens/martview
    # Follow this sequence through the pages:
    # Page 1) Make sure that the Homo_sapiens choice is selected. Hit next.
    # Page 2) Uncheck the "Limit to" box in the region choice. Then hit next.
    # Page 3) Choose the "Structures" box. 
    # Page 4) Choose GTF as the ouput.  choose gzip compression.  hit export.
    # Save as ensemblGene.gtf.gz

    # Ensembl handles random chromosomes differently than us, so we
    # strip this data.  Fortunately it just loses a couple of genes.
    # Add "chr" to front of each line in the gene data gtf file to make 
    # it compatible with our software.
    # Finally, get rid of the ".1" or ".2" after the name
    gunzip -c ensemblGene.gtf.gz \
    | grep -v ^6_DR51 \
    | grep -v ^DR51 \
    | grep -v ^DR52 \
    | grep -v ^DR53 \
    | grep -v _NT_ \
    | perl -wpe 's/^([0-9]|X|Y|Un|MT)/chr$1/ \
                 || die "Line $. doesnt start with human chrom:\n$_"' \
    | sed -e 's/\..\"/\"/g' \
    | sed -e 's/chrMT_NC_001807/chrM/' \
    > ensGene.gtf

    ssh hgwdev
    /cluster/data/hg17/bed/ensembl
    /cluster/bin/i386/ldHgGene hg17 ensGene ensGene.gtf

    # Read 33666 transcripts in 696579 lines in 1 files
    # 33666 groups 25 seqs 1 sources 4 feature types
    # 33666 gene predictions

    # ensGtp associates geneId/transcriptId/proteinId for hgPepPred and 
    # hgKnownToSuper.  Use ensMart to create it as above, except:
    # Page 3) Choose the "Features" box. In "Ensembl Attributes", check 
    # Ensembl Gene ID, Ensembl Transcript ID, Ensembl Peptide ID.  
    # Choose Text, tab-separated as the output format.  Result name ensGtp.
    # Save file as ensGtp.txt.gz
    gunzip ensGtp.txt.gz
    hgsql hg17 < ~/kent/src/hg/lib/ensGtp.sql
    # remove header line from ensGtp.txt
    echo "load data local infile 'ensGtp.txt' into table ensGtp" | hgsql -N hg17

    # Load Ensembl peptides:
    # Get them from ensembl as above in the gene section except for
    # Page 3) Choose the "Sequences" box. 
    # Page 4) Transcripts/Proteins.  Peptide.  Format = FASTA.
    # Save file as ensemblPep.fa.gz
    gunzip ensemblPep.fa.gz
    hgPepPred hg17 ensembl ensemblPep.fa

# UPDATE KNOWN GENES TABLES (DONE 11/22/04 Fan)

# Make sure the protein databases (sp041115 and proteins041115) were built first.
  
  hgsql hg17 -e "create database kgHg17B"
  
  mkdir -p /cluster/store8/kg/kgHg17B
  cd /cluster/store6/kgDB/bed
  ln -s /cluster/store8/kg/kgHg17B kgHg17B
  cd kgHg17B

    ~/src/hg/protein/KGprocess.sh kgHg17B hg17 041115

# Found gbGetSeqs changed the format of mrna.fa output file 
# (extra version number).  Updated Kgprocess.sh and manually
# re-ran the following manually:

    grep "^>" mrna.fa |awk '{print $1}' > mrna.lis

    kgGetPep 041115 > mrnaPep.fa

    hgKgMrna kgH17BTemp mrna.fa mrna.ra tight_mrna.psl ll/loc2ref \
        mrnaPep.fa ll/mim2loc ${PDB} > kgHg17BKgMrna.out 2> kgHg17BKgMrna.err

# then run Kgprocess.sh again to continue processing.

    ~/src/hg/protein/KGprocess.sh kgHg17B hg17 041115

hgsql hg17 -e "select * from chromInfo" > chromInfo.tab
getDbTabDef hg17 chromInfo >chromInfo.sql
hgsql kgHg17B <chromInfo.sql
hgsql kgHg17B -e 'load data local infile "chromInfo.tab" into table chromInfo ignore 1 lines'

# Build kgProtMap table.  This table is needed by the Proteome Browser and 
# it should be built before all the KG tables be moved from kgHg17B to hg17.

    ~/src/hg/protein/kgProtMap.sh kgHg17B hg17 041115

# Completed: 7923 of 7923 jobs
# CPU time in finished jobs:    2502923s   41715.39m   695.26h   28.97d  0.079 y
# IO & Wait Time:                175358s    2922.63m    48.71h    2.03d  0.006 y
# Average job time:                 338s       5.63m     0.09h    0.00d
# Longest job:                     2403s      40.05m     0.67h    0.03d
# Submission to last job:          7164s     119.40m     1.99h    0.08d
  
# The script ran successfully with the last message:
# Mon Nov 22 17:11:59 PST 2004 DONE =========================

# Create database hg17Kg1 to store the old KG tables, just in case.

  hgsql hg17

  create database hg17Kg1;

  alter table cgapAlias rename as hg17Kg1.cgapAlias;
  alter table cgapBiocDesc rename as hg17Kg1.cgapBiocDesc;
  alter table cgapBiocPathway rename as hg17Kg1.cgapBiocPathway;
  alter table dupSpMrna rename as hg17Kg1.dupSpMrna;
  alter table keggMapDesc rename as hg17Kg1.keggMapDesc;
  alter table keggPathway rename as hg17Kg1.keggPathway;
  alter table kgAlias rename as hg17Kg1.kgAlias;
  alter table kgProtAlias rename as hg17Kg1.kgProtAlias;
  alter table kgXref rename as hg17Kg1.kgXref;
  alter table knownGene rename as hg17Kg1.knownGene;
  alter table knownGeneLink rename as hg17Kg1.knownGeneLink;
  alter table knownGeneMrna rename as hg17Kg1.knownGeneMrna;
  alter table knownGenePep rename as hg17Kg1.knownGenePep;
  alter table mrnaRefseq rename as hg17Kg1.mrnaRefseq;
  alter table spMrna rename as hg17Kg1.spMrna;
  alter table kgProtMap rename as hg17Kg1.kgProtMap;

# After initial inspection of tables in kgHg17B, do the following
# from mySql prompt:

  alter table kgHg17B.cgapAlias rename as hg17.cgapAlias;
  alter table kgHg17B.cgapBiocDesc rename as hg17.cgapBiocDesc;
  alter table kgHg17B.cgapBiocPathway rename as hg17.cgapBiocPathway;
  alter table kgHg17B.dupSpMrna rename as hg17.dupSpMrna;
  alter table kgHg17B.keggMapDesc rename as hg17.keggMapDesc;
  alter table kgHg17B.keggPathway rename as hg17.keggPathway;
  alter table kgHg17B.kgAlias rename as hg17.kgAlias;
  alter table kgHg17B.kgProtAlias rename as hg17.kgProtAlias;
  alter table kgHg17B.kgXref rename as hg17.kgXref;
  alter table kgHg17B.knownGene rename as hg17.knownGene;
  alter table kgHg17B.knownGeneLink rename as hg17.knownGeneLink;
  alter table kgHg17B.knownGeneMrna rename as hg17.knownGeneMrna;
  alter table kgHg17B.knownGenePep rename as hg17.knownGenePep;
  alter table kgHg17B.mrnaRefseq rename as hg17.mrnaRefseq;
  alter table kgHg17B.spMrna rename as hg17.spMrna;
  alter table kgHg17B.kgProtMap rename as hg17.kgProtMap;

# Old hg17.knownGene has 43,401 entries and the new one has 44,338 entries.
# Previously:
  
    featureBits hg17 knownGene
#      65728598 bases of 2866216770 (2.293%) in intersection
# Previously, was:
#    	 63983072 bases of 2866216770 (2.232%) in intersection
  
# Connect to genome-testdb and use hgcentraltest DB.
# Update the entry in gdbPdb table:

    delete from gdbPdb where genomeDb='hg17';
    insert into gdbPdb values('hg17', 'proteins041115');

# UPDATE KGSPALIAS TABLE TO BE USED BY PB (Done 12/20/04)

    cd /cluster/data/hg17/bed/pb
    mkdir new
    cd new
    hgsql hg17 -e \
    'select kgXref.kgID, spID, alias from kgXref, kgAlias where kgXref.kgID=kgAlias.kgID' >j.tmp
    hgsql hg17 -e \
    'select kgXref.kgID, spID, alias from kgXref, kgProtAlias where kgXref.kgID=kgProtAlias.kgID'\
    >>j.tmp
    cat j.tmp|sort -u |grep -v 'kgID' >hg17.kgSpAlias.tab
    rm j.tmp

    hgsql hg17 -e 'drop table kgSpAlias';
    hgsql hg17 < ~/src/hg/lib/kgSpAlias.sql
    hgsql hg17 -e 'load data local infile "hg17.kgSpAlias.tab" into table kgSpAlias'
    gzip kgSpAlias.tab

# Create hg17GeneList.html (to be used by Google).
# This step was done 12/08/04.

    cd /cluster/data/hg17/bed
    mkdir geneList
    cd geneList
    wget -O hg17GeneList.html "http://hgwdev-fanhsu.cse.ucsc.edu/cgi-bin/hgGeneList?db=hg17"
    cp -p hg17GeneList.html /usr/local/apache/htdocs/goldenPath
# Check this html file into CVS.

# CREATE FULL TEXT INDEX FOR KNOWN GENES (DONE 1/19/2006 JK)
# This depends on the go and uniProt databases as well as 
# the kgAlias and kgProAlias tables.  The hgKgGetText takes
# about 5 minutes when the database is not too busy.  The rest
# is real quick.
     ssh hgwdev
     cd /cluster/data/hg17/bed/kgHg17F
     mkdir index
     cd index
     hgKgGetText hg17 knownGene.text
     ixIxx knownGene.text knownGene.ix knownGene.ixx
     ln -s /cluster/data/hg17/bed/kgHg17F/index/knownGene.ix /gbdb/hg17/knownGene.ix
     ln -s /cluster/data/hg17/bed/kgHg17F/index/knownGene.ixx /gbdb/hg17/knownGene.ixx



# UPDATE TABLES NEEDED BY hgGene (DONE 11/30/04 Fan)

# UPDATE BioCyc TABLES
    hgsql hg17 -e 'delete from bioCycPathway'
    hgsql hg17 -e 'delete from bioCycMapDesc'

#  Using data files sent by Peter Carp from SRI,
#  per Peter's email of 10/1/04, they don't have recent update,
#  so data files received last year are used.
  
#  Save the BioCyc Pathway name and description table as gene-pathway.dat.
#  Save the pathway data file as gene-pathway.dat.
#  Make sure there is no extra ^M at end of the lines.

# Run hgBioCyc program to generate the file bioCycPathway.tab.

	 hgBioCyc gene-pathway.dat hg17

# Load results into hg17.

    LOAD DATA local INFILE 'pathway-names.dat' into table bioCycMapDesc;
    LOAD DATA local INFILE 'bioCycPathway.tab' into table bioCycPathway;

# REBUID FOLDUTR TABLES (DONE - 2004-11-30 Fan)
# First set up directory structure and extract UTR sequence on hgwdev
    ssh hgwdev
    cd /cluster/data/hg17/bed
    mv rnaStruct rnaStruct.2004-10-04
    mkdir -p /cluster/data/hg17/bed/rnaStruct.2004-11-30
    ln -s rnaStruct.2004-11-30 rnaStruct
    cd /cluster/data/hg17/bed/rnaStruct
    mkdir -p utr3/split utr5/split utr3/fold utr5/fold
    utrFa hg17 knownGene utr3 utr3/utr.fa
    utrFa hg17 knownGene utr5 utr5/utr.fa

# Split up files and make files that define job.
    ssh kk
    cd /cluster/data/hg17/bed/rnaStruct
    faSplit sequence utr3/utr.fa 50000 utr3/split/s
    faSplit sequence utr5/utr.fa 50000 utr5/split/s
    ls -1 utr3/split > utr3/in.lst
    ls -1 utr5/split > utr5/in.lst
    cd utr3
    cat > gsub <<end
#LOOP
rnaFoldBig split/\$(path1) fold
#ENDLOOP
end
    cp gsub ../utr5

# Do cluster run for 3' UTRs
    gensub2 in.lst single gsub spec
    para create spec
    para try
    para push
# Completed: 38115 of 38115 jobs
# CPU time in finished jobs:    1101680s   18361.33m   306.02h   12.75d  0.035 y
# IO & Wait Time:                100275s    1671.25m    27.85h    1.16d  0.003 y
# Average job time:                  32s       0.53m     0.01h    0.00d
# Longest job:                     3645s      60.75m     1.01h    0.04d
# Submission to last job:          7007s     116.78m     1.95h    0.08d

# Do cluster run for 5' UTRs 
    cd ../utr5
    gensub2 in.lst single gsub spec
    para create spec
    para try
    para push
# Completed: 30524 of 30524 jobs
# CPU time in finished jobs:     116647s    1944.12m    32.40h    1.35d  0.004 y
# IO & Wait Time:                 80477s    1341.28m    22.35h    0.93d  0.003 y
# Average job time:                   6s       0.11m     0.00h    0.00d
# Longest job:                     2449s      40.82m     0.68h    0.03d
# Submission to last job:          3386s      56.43m     0.94h    0.04d

# Load database
    ssh hgwdev
    cd /cluster/data/hg17/bed/rnaStruct/utr5
    hgLoadRnaFold hg17 foldUtr5 fold
# Parsed 30525 files
# Warning: load of foldUtr5 did not go as planned: 30525 record(s), 2 row(s) skipped, 0 warning(s) loading 
./foldUtr5.tab

    cd ../utr3
    hgLoadRnaFold hg17 foldUtr3 fold
# Parsed 38115 files
#Warning: load of foldUtr3 did not go as planned: 38115 record(s), 2 row(s) skipped, 0 warning(s) loading 
./foldUtr3.tab

# Clean up
    rm -r split fold err batch.bak
    cd ../utr5
    rm -r split fold err batch.bak

# UPDATE GENE SORTER TABLES (AKA: FAMILY BROWSER) (DONE - 2004-11-29 - Fan)
#	This should be done after knownGene tables are complete from known gene
#	process.
#
# Cluster together various alt-splicing isoforms.
#	Creates the knownIsoforms and knownCanonical tables
ssh hgwdev
mkdir /cluster/data/hg17/bed/geneSorter.2004-11-24
# remove old symbolic link
rm /cluster/data/hg17/bed/geneSorter
ln -s /cluster/data/hg17/bed/geneSorter.2004-11-24 \
	/cluster/data/hg17/bed/geneSorter
cd /cluster/data/hg17/bed/geneSorter
hgClusterGenes hg17 knownGene knownIsoforms knownCanonical

# Extract peptides from knownGenes into fasta file
# and create a blast database out of them.
mkdir /cluster/data/hg17/bed/geneSorter/blastp
cd /cluster/data/hg17/bed/geneSorter/blastp
pepPredToFa hg17 knownGenePep known.faa
#	You may need to build this binary in src/hg/near/pepPredToFa
/scratch/blast/formatdb -i known.faa -t known -n known
#	This command is in /projects/compbio/bin/$MACH/formatdb

# Copy over database to bluearc
rm -fr /cluster/bluearc/hg17/blastp
mkdir -p /cluster/bluearc/hg17/blastp
cp -p /cluster/data/hg17/bed/geneSorter/blastp/known.* \
	/cluster/bluearc/hg17/blastp

#	Had to pick up a new blastall binary (2004-06-15)
#	Our old one would no longer run on our systems that have
#	updated Linux versions
mkdir /cluster/bluearc/blast2210
cd /cluster/bluearc/blast2210
wget --timestamping \
    ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/2.2.10/blast-2.2.10-ia32-linux.tar.gz
wget --timestamping \
    ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/2.2.10/ChangeLog.txt
wget --timestamping \
    ftp://ftp.ncbi.nlm.nih.gov/blast/executables/release/2.2.10/ReleaseNotes.txt
tar xvzf blast-2.2.10-ia32-linux.tar.gz

# Split up fasta file into bite sized chunks for cluster
cd /cluster/data/hg17/bed/geneSorter/blastp
mkdir split
faSplit sequence known.faa 8000 split/kg

# Make parasol run directory
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/self
cd /cluster/data/hg17/bed/geneSorter/blastp/self
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
-p blastp -d /cluster/bluearc/hg17/blastp/known -i $1 -o $2 \
-e 0.01 -m 8 -b 1000
'_EOF_'
    # << keep emacs happy
chmod +x blastSome

# Make gensub2 file
cat  << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#	'ls ../../split/*.fa' is too much, hence the echo
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
# Wait a couple of minutes, and do a para check,  if all is good
# then do a
para push
# This should finish in ~15 minutes if the cluster is free.
Completed: 7748 of 7748 jobs
CPU time in finished jobs:     191136s    3185.59m    53.09h    2.21d  0.006 y
IO & Wait Time:                 66703s    1111.72m    18.53h    0.77d  0.002 y
Average job time:                  33s       0.55m     0.01h    0.00d
Longest job:                      370s       6.17m     0.10h    0.00d
Submission to last job:           747s      12.45m     0.21h    0.01d

# Load into database.  This takes about 30 minutes
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/self/run/out
time hgLoadBlastTab hg17 knownBlastTab *.tab
# Scanning through 7748 files
# Loading database with 12810133 rows
# 306.480u 54.190s 26:35.50 22.6% 0+0k 0+0io 206pf+0w

cd /cluster/data/hg17/bed/geneSorter
# Create table that maps between known genes and RefSeq
hgMapToGene hg17 refGene knownGene knownToRefSeq
#	may need to build this command in src/hg/near/hgMapToGene
#	hgsql -e "select count(*) from knownToRefSeq;" hg17
#	row count changed 37611

# Create table that maps between known genes and LocusLink
hgsql --skip-column-names -e "select mrnaAcc,locusLinkId from refLink" hg17 \
	> refToLl.txt
hgMapToGene hg17 refGene knownGene knownToLocusLink -lookup=refToLl.txt
#	hgsql -e "select count(*) from knownToLocusLink;" hg17
#	row count changed to 37611

# Create table that maps between known genes and Pfam domains
hgMapViaSwissProt hg17 knownGene name proteinID Pfam knownToPfam
#	hgsql -e "select count(*) from knownToPfam;" hg17
#	row count changed to 36302

# Create table to map between known genes and GNF Atlas2
# expression data.
    hgMapToGene hg17 gnfAtlas2 knownGene knownToGnfAtlas2 '-type=bed 12'
#	hgsql -e "select count(*) from knownToGnfAtlas2;" hg17
#	row count changed to 36373

# Create expression distance table - takes about an hour
    hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
    	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
	-lookup=knownToGnfAtlas2 &
# Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
# Got 36373 unique elements in hgFixed.gnfHumanAtlas2MedianRatio

#	hgsql -e "select count(*) from gnfAtlas2Distance;" hg17
#	row count changed to 36373000

# Create a table that maps between known genes and 
# the nice affy expression data.
hgMapToGene "-type=bed 12" hg17 affyUclaNorm knownGene knownToU133
#	hgsql -e "select count(*) from knownToU133;" hg17
#	row count changed to 37299

# Create expression distance table.  This will take about 2.5 hours
cd /tmp
cp -p ~/kent/src/hg/near/hgExpDistance/affyUcla.weight .
time hgExpDistance hg17 affyUclaNorm affyUclaExp knownExpDistance \
	-weights=affyUcla.weight -lookup=knownToU133 &
# Have 43039 elements in affyUclaNorm
# 211 genes, 42 weights, 26.500000 total wieght
# Got 37299 unique elements in affyUclaNorm
# 8212.320u 217.310s 2:38:07.84 88.8%     0+0k 0+0io 267pf+0w
# Create table that maps between known genes and 
# the GNF data.
cd /tmp
hgMapToGene hg17 affyU95 knownGene knownToU95
#	row count changed to 18791
#	hgFixed.gnfHumanU95Exps argument is unused, no need to exist
hgExpDistance hg17 hgFixed.gnfHumanU95MedianRatio \
	hgFixed.gnfHumanU95Exps gnfU95Distance  -lookup=knownToU95 &
# Have 11545 elements in hgFixed.gnfHumanU95MedianRatio
# Got 17682 unique elements in hgFixed.gnfHumanU95MedianRatio
#	row count changed to 17682000 

# Create known gene mapping table and expression distance tables
# for GNF Atlas 2.  (The hgExpDistance takes only 10 minutes.)

hgMapToGene hg17 affyGnf1h knownGene knownToGnf1h
hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
	-lookup=knownToGnf1h &
# Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
# Got 10273 unique elements in hgFixed.gnfHumanAtlas2MedianRatio

cd /cluster/data/hg17/bed/geneSorter
hgMapToGene hg17 affyU133Plus2 knownGene knownToU133Plus2
#	row count changed to 40015 

# Make sure that GO database is up to date.
# UPDATE GO DATABASE (DONE 11/24/04 Fan)

# Download the terms and make the database.
ssh hgwdev
mkdir /cluster/store1/geneOntology/20041124
cd /cluster/store1/geneOntology/20041124

wget --timestamping http://www.godatabase.org/dev/database/archive/latest/go_200411-assocdb-data.gz

hgsql mysql <<end
create database go041124;
end
zcat go_*data.gz | hgsql go041124

wget -timestamping ftp://ftp.geneontology.org/pub/go/gene-associations/gene_association.goa_sptr.gz

wget --timestamping ftp://ftp.geneontology.org/pub/go/gene-associations/gene_association.goa_uniprot.gz

zcat gene_association.goa_uniprot.gz | hgGoAssociation go041124 goaPart stdin 
# Passed 4502016 of 5291097 of 5291097, 85.09%

# Ask sys-admin to switch the database pointer go to point to go041124.

cd /cluster/data/hg17/bed/geneSorter

XXX - DO NOT YET HAVE ensGene table - must wait on Ensembl to release that
XXX - have not created the knownToEnsembl table yet - 2004-07-15 - Hiram
# Create knownToEnsembl column
hgMapToGene hg17 ensGene knownGene knownToEnsembl
#	table row count went from previous version: 36068 to 38251

# Make knownToCdsSnp table (Heather did this table, Nov 29, 2004)
  ssh hgwdev
  nice hgMapToGene hg17 snp knownGene knownToCdsSnp -all -cds
# row count 168336
# approx. 5 minutes running time

# Make C. elegans ortholog column using blastp on wormpep.
# First make C. elegans protein database and copy it to cluster/bluearc
# There was no /cluster/bluearc/ce1/blastp, so get the latest wormpep from Sanger
cd /cluster/data/ce1/bed/blastp
mkdir old
cp -p * old
wget --timestamp ftp://ftp.sanger.ac.uk/pub/databases/wormpep/wormpep134/wormpep134
mv wormpep134 wormPep.faa
formatdb -i wormPep.faa -t wormPep -n wormPep
#copy them to /cluster/bluearc
ssh kkr1u00
mkdir -p /cluster/bluearc/ce1/blastp
cp /cluster/data/ce1/bed/blastp/wormPep.p?? /cluster/bluearc/ce1/blastp

#	The blast jobs below can be run on the kk or kk9 clusters
# Create the ceBlastTab
ssh kk9
mkdir /cluster/data/hg17/bed/geneSorter/blastp/ce1
cd /cluster/data/hg17/bed/geneSorter/blastp/ce1
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
	-p blastp -d /cluster/bluearc/ce1/blastp/wormPep \
	-i $1 -o $2 -e 0.01 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
#	Only takes 10 minutes on an idle cluster
# Completed: 7748 of 7748 jobs
# CPU time in finished jobs:      33235s     553.91m     9.23h    0.38d  0.001 y
# IO & Wait Time:                 19891s     331.52m     5.53h    0.23d  0.001 y
# Average job time:                   7s       0.11m     0.00h    0.00d
# Longest job:                       68s       1.13m     0.02h    0.00d
# Submission to last job:           653s      10.88m     0.18h    0.01d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/ce1/run/out
hgLoadBlastTab hg17 ceBlastTab -maxPer=1 *.tab
#	row count changed to 28252

# Make mouse ortholog column using blastp on mouse known genes.
# First make mouse protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This already exists.  See makeMm5.doc for procedure
#	the directory: /cluster/bluearc/scratch/mus/mm5/blastp should have data

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/mm5
cd /cluster/data/hg17/bed/geneSorter/blastp/mm5
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
	-p blastp -d /cluster/bluearc/scratch/mus/mm5/blastp/known \
	-i $1 -o $2 -e 0.001 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#	this echo trick is used because otherwise the command line is
#	too long and you can not do a simple ls
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7748 of 7748 jobs
# CPU time in finished jobs:     141842s    2364.04m    39.40h    1.64d  0.004 y
# IO & Wait Time:                 52251s     870.85m    14.51h    0.60d  0.002 y
# Average job time:                  25s       0.42m     0.01h    0.00d
# Longest job:                      254s       4.23m     0.07h    0.00d
# Submission to last job:           540s       9.00m     0.15h    0.01d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/mm5/run/out
hgLoadBlastTab hg17 mmBlastTab -maxPer=1 *.tab
# Scanning through 7748 files
#	row count changed to 37549 

# Make rat ortholog column using blastp on rat known genes.
# First make rat protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This already exists.  See makeRn3.doc for procedure.
#	Files were put in this directory: /cluster/bluearc/rn3/blastp/

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/rn3
cd /cluster/data/hg17/bed/geneSorter/blastp/rn3
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
    -p blastp -d /cluster/bluearc/rn3/blastp/known \
    -i $1 -o $2 -e 0.001 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#	this echo trick is used because otherwise the command line is
#	too long and you can not do a simple ls
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7748 of 7748 jobs
# CPU time in finished jobs:      31786s     529.77m     8.83h    0.37d  0.001 y
# IO & Wait Time:                 25795s     429.91m     7.17h    0.30d  0.001 y
# Average job time:                   7s       0.12m     0.00h    0.00d
# Longest job:                       75s       1.25m     0.02h    0.00d
# Submission to last job:           157s       2.62m     0.04h    0.00d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/rn3/run/out
hgLoadBlastTab hg17 rnBlastTab -maxPer=1 *.tab
# Scanning through 7748 files
#Loading database with 26133 rows

# Make Danio rerio (zebrafish) ortholog column using blastp on Ensembl.
# First make protein database and copy it to cluster/bluearc
# if it doesn't exist already
# NOTE: data used to reside in /cluster/bluearc/dr1/blastp
mv /cluster/bluearc/dr1/blastp /cluster/bluearc/danRer1/blastp
#	the directory: /cluster/bluearc/danRer1/blastp should have data

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/danRer1
cd /cluster/data/hg17/bed/geneSorter/blastp/danRer1
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
	-p blastp -d /cluster/bluearc/danRer1/blastp/ensembl \
	-i $1 -o $2 -e 0.005 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7748 of 7748 jobs
# CPU time in finished jobs:     102324s    1705.39m    28.42h    1.18d  0.003 y
# IO & Wait Time:                 47203s     786.72m    13.11h    0.55d  0.001 y
# Average job time:                  19s       0.32m     0.01h    0.00d
# Longest job:                      230s       3.83m     0.06h    0.00d
# Submission to last job:           427s       7.12m     0.12h    0.00d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/danRer1/run/out
hgLoadBlastTab hg17 drBlastTab -maxPer=1 *.tab
# Loading database with 33852 rows

# Make Saccharomyces cerevisiae (yeast) ortholog column using blastp on RefSeq.
# First make protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This is already done, see makeMm3.doc for procedure
#	the directory: /cluster/bluearc/sc1/blastp should have data

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/sc1
cd /cluster/data/hg17/bed/geneSorter/blastp/sc1
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
	-p blastp -d /cluster/bluearc/sc1/blastp/sgd \
	-i $1 -o $2 -e 0.01 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7748 of 7748 jobs
# CPU time in finished jobs:      20983s     349.72m     5.83h    0.24d  0.001 y
# IO & Wait Time:                 25513s     425.21m     7.09h    0.30d  0.001 y
# Average job time:                   6s       0.10m     0.00h    0.00d
# Longest job:                       37s       0.62m     0.01h    0.00d
# Submission to last job:           106s       1.77m     0.03h    0.00d
# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/sc1/run/out
hgLoadBlastTab hg17 scBlastTab -maxPer=1 *.tab
# Loading database with 18489 rows

# Make Drosophila melanagaster ortholog column using blastp on FlyBase.
# First make SwissProt protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This is already done, see makeMm3.doc for procedure
#	the directory: /cluster/bluearc/dm1/blastp should have data

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/dm1
cd /cluster/data/hg17/bed/geneSorter/blastp/dm1
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
	-p blastp -d /cluster/bluearc/dm1/blastp/flyBase \
	-i $1 -o $2 -e 0.01 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7748 of 7748 jobs
# CPU time in finished jobs:      83377s    1389.62m    23.16h    0.97d  0.003 y
# IO & Wait Time:                 39913s     665.21m    11.09h    0.46d  0.001 y
# Average job time:                  16s       0.27m     0.00h    0.00d
# Longest job:                      167s       2.78m     0.05h    0.00d
# Submission to last job:           365s       6.08m     0.10h    0.00d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/dm1/run/out
hgLoadBlastTab hg17 dmBlastTab -maxPer=1 *.tab
# Loading database with 30067 rows

# update knownToHInv table
# Verified that there is now new release of HInv data.
hgMapToGene hg17 HInvGeneMrna knownGene knownToHInv
# count changed to 33236

#### BUILD Ensembl cross-reference table, ensemblXref3 (DONE - 2004-11-30 - Fan)

    # Get the ensembl gene/protein cross-reference data from
    # http://www.ensembl.org/Homo_sapiens/martview
    # Follow this sequence through the pages:
    # Page 1) Make sure that the Homo_sapiens choice is selected. Hit next.
    # Page 2) Uncheck the "Limit to" box in the region choice. Then hit next.
    # Page 3) Choose the "Feature" box, select gene, transcript, protein,
              SPTrEMBL ID, SWISSPROT ID, and SWISSPROT AC
    # Page 4) Choose "Text, tab separated".  choose gzip compression.  hit export.
    # Save as ensXref.txt

    sed ensXref.txt -e 's/\./\t/g' > ensemblXref3.tab

    hgsql hg17 -e "drop table ensemblXref3"
    hgsql hg17 < ~/src/hg/lib/ensemblXref3.sql

    hgsql hg17 -e 'load data local infile "ensemblXref3.tab" into table ensemblXref3 ignore 1 lines'

#### BUILD SUPERFAMILY RELATED TABLES (DONE - 2004-11-30 - Fan)

# Download Superfamily data files and build the Superfamily DB
# from supfam.mrc-lmb.cam.ac.uk

    mkdir /cluster/store8/superfamily/041128
    ln -s /cluster/store8/superfamily/041128 /cluster/data/superfamily/041128
    cd /cluster/data/superfamily/041128
# ftp over the following two files:
ass_28-Nov-2004.tab.gz
      supfam_28-Nov-2004.sql.gz
    
    gzip -d *.gz
# Load the Superfamily database
    hgsql hg17 -e "create database superfam041128"
    hgsql superfam041128 < supfam_28-Nov-2004.sql
# This may take about an hour.

# Make sure to add an index on id of the des table of superfam041128.
    hgsql superfam041128 -e "create index id on des(id);"

    hgsql superfam041128 < ~/src/hg/lib/sfAssign.sql
    hgsql superfam041128 -e 'load data local infile "ass_28-Nov-2004.tab" into table 
superfam041128.sfAssign;'

# Build or rebuild Superfamily track and create sf tables needed for PB

   hgsql hg17 < ~/src/hg/lib/sfAssign.sql

   cd /cluster/data/superfamily/041128  
   hgsql hg17 -e 'load data local infile "ass_28-Nov-2004.tab" into table hg17.sfAssign;'

# If hg17.sfDes already exists, drop it.

   hgsql superfam041128 -e "select * from des" >sfDes.tab
   hgsql hg17 < ~/src/hg/lib/sfDes.sql
   hgsql hg17 -e 'load data local infile "sfDes.tab" into table hg17.sfDes ignore 1 lines;'

# If hg17.superfamily already exists, drop it.
   cd /cluster/data/hg17/bed
   mkdir /cluster/data/hg17/sf.2004-1128
   ln -s sf.2004-1128 sf
   hgSuperfam hg17 > sf.log

# It is normal that many proteins does not have corresponding Superfamily entries.

# If hg17.sfDescription exists, drop it.

   hgsql hg17 < ~/src/hg/lib/sfDescription.sql
   hgsql hg17 -e 'LOAD DATA local INFILE "sfDescription.tab"  into table hg17.sfDescription;'

# Finally, load the superfamily table.

   hgLoadBed hg17 superfamily superfamily.tab -tab

# Create knownToSuperfamily table
# Note hs is changed into ht for this Superfamily release.
   
   cat /cluster/data/superfamily/041128/ass_28-Nov-2004.tab \
   | hgKnownToSuper hg17 hs stdin
# created 25287 rows in knownToSuper

### HG17 PROTEOME BROWSER TABLES RE-BUILD ####  (DONE - 2004-12-01 - Fan)
# These are instructions for rebuilding tables 
# needed for the Proteome Browser.  
# DON'T START THESE UNTIL TABLES FOR KNOWN GENES AND kgProtMap table
# ARE REBUILT.  
# This update is based on proteins DBs dated 041115.

# Create the working directory

    ssh hgwdev
    mv /cluster/data/hg17/bed/pb /cluster/data/hg17/bed/pb.2004-06-11
    mkdir /cluster/data/hg17/bed/pb.2004-12-01
    cd /cluster/data/hg17/bed
    ln -s /cluster/data/hg17/bed/pb.2004-12-01 pb

# Move the existing PB tables by:

hgsql hg17
create database hg17Sav2;

alter table hg17.pepCCntDist rename as hg17Sav2.pepCCntDist;
alter table hg17.pepExonCntDist rename as hg17Sav2.pepExonCntDist;
alter table hg17.pepHydroDist rename as hg17Sav2.pepHydroDist;
alter table hg17.pepIPCntDist rename as hg17Sav2.pepIPCntDist;
alter table hg17.pepMolWtDist rename as hg17Sav2.pepMolWtDist;
alter table hg17.pepMwAa rename as hg17Sav2.pepMwAa;
alter table hg17.pepPi rename as hg17Sav2.pepPi;
alter table hg17.pepPiDist rename as hg17Sav2.pepPiDist;
alter table hg17.pepResDist rename as hg17Sav2.pepResDist;

alter table hg17.pbAaDistA rename as hg17Sav2.pbAaDistA;
alter table hg17.pbAaDistC rename as hg17Sav2.pbAaDistC;
alter table hg17.pbAaDistD rename as hg17Sav2.pbAaDistD;
alter table hg17.pbAaDistE rename as hg17Sav2.pbAaDistE;
alter table hg17.pbAaDistF rename as hg17Sav2.pbAaDistF;
alter table hg17.pbAaDistG rename as hg17Sav2.pbAaDistG;
alter table hg17.pbAaDistH rename as hg17Sav2.pbAaDistH;
alter table hg17.pbAaDistI rename as hg17Sav2.pbAaDistI;
alter table hg17.pbAaDistK rename as hg17Sav2.pbAaDistK;
alter table hg17.pbAaDistL rename as hg17Sav2.pbAaDistL;
alter table hg17.pbAaDistM rename as hg17Sav2.pbAaDistM;
alter table hg17.pbAaDistN rename as hg17Sav2.pbAaDistN;
alter table hg17.pbAaDistP rename as hg17Sav2.pbAaDistP;
alter table hg17.pbAaDistQ rename as hg17Sav2.pbAaDistQ;
alter table hg17.pbAaDistR rename as hg17Sav2.pbAaDistR;
alter table hg17.pbAaDistS rename as hg17Sav2.pbAaDistS;
alter table hg17.pbAaDistT rename as hg17Sav2.pbAaDistT;
alter table hg17.pbAaDistV rename as hg17Sav2.pbAaDistV;
alter table hg17.pbAaDistW rename as hg17Sav2.pbAaDistW;
alter table hg17.pbAaDistY rename as hg17Sav2.pbAaDistY;
alter table hg17.pbAnomLimit rename as hg17Sav2.pbAnomLimit;
alter table hg17.pbResAvgStd rename as hg17Sav2.pbResAvgStd;
alter table hg17.pbStamp rename as hg17Sav2.pbStamp;

quit

# Define pep* tables in hg17 DB

	cat ~/kent/src/hg/lib/pep*.sql > pepAll.sql

#  First edit out pepPred table definition, then

	hgsql hg17 < pepAll.sql

# Build the pepMwAa table

  hgsql proteins041115 -e "select info.acc, molWeight, aaSize from sp041115.info, sp041115.accToTaxon where 
accToTaxon.taxon=9606 and accToTaxon.acc = info.acc" > pepMwAa.tab

hgsql hg17 <<end
load data local infile "pepMwAa.tab" into table hg17.pepMwAa ignore 1 lines;
end

o Build the pepPi table

  hgsql proteins041115 -e "select info.acc from sp041115.info, sp041115.accToTaxon where 
accToTaxon.taxon=9606 and accToTaxon.acc = info.acc" > protAcc.lis

  pbCalPi protAcc.lis sp041115 pepPi.tab

hgsql hg17 <<end
load data local infile "pepPi.tab" into table hg17.pepPi;
end

# Calculate and load pep distributions

  pbCalDist sp041115 proteins041115 9606 hg17 >pbCalDist.out

    cat pbCalDist.out
    wc  pbCalDist.out

    hgsql hg17

    load data local infile "pepExonCntDist.tab" into table hg17.pepExonCntDist;
    load data local infile "pepCCntDist.tab" into table hg17.pepCCntDist;
    load data local infile "pepHydroDist.tab" into table hg17.pepHydroDist;
    load data local infile "pepMolWtDist.tab" into table hg17.pepMolWtDist;
    load data local infile "pepResDist.tab" into table hg17.pepResDist;
    load data local infile "pepIPCntDist.tab" into table hg17.pepIPCntDist;
    load data local infile "pepPiDist.tab" into table hg17.pepPiDist;
    quit

# Calculate frequency distributions

    pbCalResStd 041115 9606 hg17

# Create pbAnomLimit and pbResAvgStd tables

   hgsql hg17 < ~/src/hg/lib/pbAnomLimit.sql
   hgsql hg17 < ~/src/hg/lib/pbResAvgStd.sql

   hgsql hg17 -e 'load data local infile "pbResAvgStd.tab" into table hg17.pbResAvgStd;'
   hgsql hg17 -e 'load data local infile "pbAnomLimit.tab" into table hg17.pbAnomLimit;'

# Create pbStamp table for PB
   
  hgsql hg17 < ~/src/hg/lib/pbStamp.sql
  hgsql hg17Sav2 -e 'select * from pbStamp' > pbStamp.tab

  hgsql hg17 -e 'load data local infile "pbStamp.tab" into table hg17.pbStamp ignore 1 lines;'

# Adjust drawing parameters for Proteome Browser stamps

  Now invoke Proteome Browser and adjust various drawing parameters
  (mostly the ymax of each stamp) if necessary, by updating the 
  pbStamp.tab file and then delete and reload the pbStamp table. 

# Perform preliminary review of Proteome Browser for hg17, then
  notify QA for formal review.

####  Blat knownGene proteins to determine exons  (braney DONE 12/11/04)
    ssh hgwdev
    cd /cluster/data/hg17/bed
    mkdir blat.hg17KG.2004-12-08
    rm blat.hg17KG
    ln -s  blat.hg17KG.2014-12-08 blat.hg17KG
    cd blat.hg17KG
    pepPredToFa hg17 knownGenePep known.fa
    ssh kk
    cd /cluster/data/hg17/bed/blat.hg17KG
    cat << '_EOF_' > blatSome
#!/bin/csh -fe
/cluster/bin/i386/blat -t=dnax -q=prot -out=pslx $1 $2 $3
'_EOF_'
    cat << '_EOF_' > blatGsub
#LOOP
blatSome $(path1) {check in line $(path2)} {check out line psl/$(root1)/$(root2).psl}
#ENDLOOP
'_EOF_'
    # << keep emacs happy
    chmod +x blatSome
    ls -1S /scratch/hg/gs.18/build35/bothMaskedNibs/*.nib > human.lst
    mkdir kgfa
    cd kgfa
    faSplit sequence ../known.fa 3010 kg
    cd ..
    ls -1S kgfa/*.fa > kg.lst
    gensub2 human.lst kg.lst blatGsub blatSpec
    mkdir psl
    cd psl
    foreach i (`cat ../human.lst`)
	mkdir `basename $i .nib`
    end
    cd ..
    para create blatSpec
    para push

# Completed: 134130 of 134136 jobs
# Crashed: 6 jobs
# CPU time in finished jobs:   29801114s  496685.23m  8278.09h  344.92d  0.945 y
# IO & Wait Time:               1983513s   33058.55m   550.98h   22.96d  0.063 y
# Average job time:                 237s       3.95m     0.07h    0.00d
# Longest job:                    63306s    1055.10m    17.59h    0.73d
# Submission to last job:        169384s    2823.07m    47.05h    1.96d

# did 6 crashed jobs on small cluster
    ssh eieio
    cd /cluster/data/hg17/bed/blat.hg17KG
    pslSort dirs raw.psl /tmp psl/*
    pslReps -nohead -minCover=0.9 -minAli=0.9 raw.psl cooked.psl /dev/null
    pslUniq cooked.psl hg17KG.psl
    pslxToFa hg17KG.psl hg17KG_ex.fa -liftTarget=genome.lft -liftQuery=protein.lft
    kgName hg17 hg17KG.psl blastKGRef01
    cut -f 10 hg17KG.psl > kgName.lst
    faSomeRecords known.fa kgName.lst hg17KG.fa
    hgPepPred hg17 generic blastKGPep01 hg17KG.fa

    ssh hgwdev
    cd /cluster/data/hg17/bed/blat.hg17KG
    hgsql hg17 < ~/kent/src/hg/lib/blastRef.sql
    echo "rename table blastRef to blastKGRef01" | hgsql hg17
    echo "load data local infile 'blastKGRef01' into table blastKGRef01" | hgsql hg17

#### TIGR GENE INDEX (DONE 2004-12-04 Fan)
    mkdir -p /cluster/data/hg17/bed/tigr
    cd /cluster/data/hg17/bed/tigr
    wget -timestamp ftp://ftp.tigr.org/pub/data/tgi/Homo_sapiens/TGI_track_HumanGenome_build35.tgz
    tar xvzf TGI*.tgz

    foreach f (*cattle*)
      set f1 = `echo $f | sed -e 's/cattle/cow/g'`
      mv $f $f1
    end

    foreach o (mouse cow human pig rat)
      echo $o
      setenv O $o
      foreach f (chr*_$o*s)
        tail +2 $f | perl -wpe 's /THC/TC/; s/(TH?C\d+)/$ENV{O}_$1/;' > $f.gff
      end
    end

    ssh hgwdev
    cd /cluster/data/hg17/bed/tigr
    hgsql hg17 -e "drop table tigrGeneIndex"
    hgsql hg17 < ~/kent/src/hg/lib/tigrGeneIndex.sql

    foreach f (*.gff)
        echo Processing $f ...
        /cluster/home/fanhsu/bin/i386/ldHgGene -oldTable -exon=TC hg17 tigrGeneIndex $f
        hgsql hg17 -e "select count(*) from tigrGeneIndex"
    end
    # Total of 401322 entries created in tigrGeneIndex table.

    hgsql hg17 -e "update tigrGeneIndex set cdsStart = txStart;"
    hgsql hg17 -e "update tigrGeneIndex set cdsEnd = txEnd;"

    checkTableCoords hg17 tigrGeneIndex

    gzip *.gff *TCs

# BLASTZ FOR ZEBRAFISH (danRer2) (DONE, 2004-12-09, hartera)
    ssh kkr1u00
    # blastz requires lineage-specific repeats
    # Treat all repeats as lineage-specific.
 # /iscratch/i/gs.18/build35/linSpecRep.notInZebrafish exists (makeDanRer1.doc)
    mkdir -p /iscratch/i/danRer2/linSpecRep.notInHuman
    foreach f (/iscratch/i/danRer2/rmsk/chr*.fa.out)
      cp -p $f /iscratch/i/danRer2/linSpecRep.notInHuman/$f:t:r:r.out.spec
    end
    iSync

    ssh kk
    mkdir -p /cluster/data/hg17/bed/blastz.danRer2.2004-12-08
    ln -s /cluster/data/hg17/bed/blastz.danRer2.2004-12-08 \
          /cluster/data/hg17/bed/blastz.danRer2
    cd /cluster/data/hg17/bed/blastz.danRer2
    # Set L=6000 and abridge repeats - these are the same parameters used
    # for hg16 and Fugu and similar to those for hg16-galgal2
                                                                                
    cat << '_EOF_' > DEF
# human (hg17) vs zebrafish (danRer2)
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin
                                                                                
ALIGN=blastz-run
BLASTZ=blastz
                                                                                
# Reuse parameters from hg16-fr1.
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_Q=/cluster/data/blastz/HoxD55.q
BLASTZ_ABRIDGE_REPEATS=1
                                                                                
# TARGET: Human (hg17)
SEQ1_DIR=/iscratch/i/gs.18/build35/bothMaskedNibs
SEQ1_RMSK=
SEQ1_FLAG=
SEQ1_SMSK=/iscratch/i/gs.18/build35/linSpecRep.notInZebrafish
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
                                                                                
# QUERY: zebrafish (danRer2)
SEQ2_DIR=/iscratch/i/danRer2/nib/
SEQ2_RMSK=
SEQ2_FLAG=
SEQ2_SMSK=/iscratch/i/danRer2/linSpecRep.notInHuman
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=10000000
SEQ2_LAP=0
                                                                                
BASE=/cluster/data/hg17/bed/blastz.danRer2
                                                                                
DEF=$BASE/DEF
RAW=$BASE/raw
CDBDIR=$BASE
SEQ1_LEN=$BASE/S1.len
SEQ2_LEN=$BASE/S2.len

#DEBUG=1
'_EOF_'
    # << this line keeps emacs coloring happy
    chmod +x DEF
    # Save the DEF file in the current standard place
    cp DEF ~angie/hummus/DEF.hg17-danRer2.2004-12-08
    # prepare first cluster run
    ssh kk
    cd /cluster/data/hg17/bed/blastz.danRer2
    bash # if a csh/tcsh user
    . ./DEF
    /cluster/data/hg17/jkStuff/BlastZ_run0.sh
    cd run.0
    para try, check, push, check ...etc.
# para time
# Completed: 58993 of 58993 jobs
# CPU time in finished jobs:   19583036s  326383.93m  5439.73h  226.66d  0.621 y
# IO & Wait Time:                471090s    7851.50m   130.86h    5.45d  0.015 y
# Average job time:                 340s       5.67m     0.09h    0.00d
# Longest job:                      885s      14.75m     0.25h    0.01d
# Submission to last job:         78245s    1304.08m    21.73h    0.91d

    ssh kki
    cd /cluster/data/hg17/bed/blastz.danRer2
    bash # if a csh/tcsh user
    . ./DEF
    /cluster/data/hg17/jkStuff/BlastZ_run1.sh
    cd run.1
    para try, check, push, etc ...
# para time
# Completed: 341 of 341 jobs
# CPU time in finished jobs:        789s      13.14m     0.22h    0.01d  0.000 y
# IO & Wait Time:                  2992s      49.87m     0.83h    0.03d  0.000 y
# Average job time:                  11s       0.18m     0.00h    0.00d
# Longest job:                       34s       0.57m     0.01h    0.00d
# Submission to last job:           391s       6.52m     0.11h    0.00d

    #   Third cluster run to convert lav's to axt's
    ssh kki
    cd /cluster/data/hg17/bed/blastz.danRer2
    mkdir axtChrom
    # a new run directory
    mkdir run.2
    cd run.2
cat << '_EOF_' > do.csh
#!/bin/csh
cd $1
cat `ls -1 *.lav | sort -g` \
| lavToAxt stdin /iscratch/i/gs.18/build35/bothMaskedNibs \
 /iscratch/i/danRer2/nib stdout \
| axtSort stdin $2
'_EOF_'
    # << this line makes emacs coloring happy
    chmod a+x do.csh
    cat << '_EOF_' > gsub
#LOOP
./do.csh {check in exists $(path1)} {check out line+ /cluster/data/hg17/bed/blastz.danRer2/axtChrom/$(root1).axt}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy
    \ls -1Sd ../lav/chr* > chrom.list
    gensub2 chrom.list single gsub jobList
    wc -l jobList
    head jobList
    para create jobList
    para try, check, push, check,...
# para time
# Completed: 45 of 46 jobs
# Crashed: 1 jobs
# CPU time in finished jobs:         99s       1.64m     0.03h    0.00d  0.000 y
# IO & Wait Time:                   862s      14.37m     0.24h    0.01d  0.000 y
# Average job time:                  21s       0.36m     0.01h    0.00d
# Longest job:                       92s       1.53m     0.03h    0.00d
# Submission to last job:           456s       7.60m     0.13h    0.01d
    # crashed job: chr6_hla_hap1.axt is empty - has no alignments
    # translate sorted axt files into psl
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.danRer2
    mkdir -p pslChrom
    set tbl = "blastzDanRer2"
    foreach f (axtChrom/chr*.axt)
      set c=$f:t:r
      echo "Processing chr $c"
      /cluster/bin/i386/axtToPsl $f S1.len S2.len pslChrom/${c}_${tbl}.psl
    end
                                                                               
    # Load database tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.danRer2/pslChrom
                                                                               
    foreach f (./*.psl)
      /cluster/bin/i386/hgLoadPsl -noTNameIx hg17 $f
      echo "$f Done"
    end
# try different parameters for blastz with chr1 of hg17
# featureBits -chrom=chr1 hg17 refGene:cds blastzDanRer1 -enrichment
# refGene:cds 1.301%, blastzDanRer1 3.934%, both 0.874%, cover 67.23%, 
# enrich 17.09x
# H=2000, Y=3400, L=6000, K=2200 and HoxD55.q scoring matrix
# featureBits -chrom=chr1 hg17 refGene:cds blastzDanRer2 -enrichment
# refGene:cds 1.301%, blastzDanRer2 3.845%, both 0.879%, cover 67.55%, 
# enrich 17.57x
# same parameters as above but L=8000
# featureBits -chrom=chr1 hg17 refGene:cds blastzDanRer2L8k -enrichment
# refGene:cds 1.301%, blastzDanRer2L8k 2.309%, both 0.778%, cover 59.81%, 
# enrich 25.91x
# enrichment went up but coverage dropped quite a bit.
# Default parameters with H=2000
# featureBits -chrom=chr1 hg17 refGene:cds blastzDanRer2Default -enrichment
# refGene:cds 1.301%, blastzDanRer2Default 1.701%, both 0.846%, cover 65.04%, 
# enrich 38.24x
# same as first run but with no Y option set (default Y)
# featureBits -chrom=chr1 hg17 refGene:cds blastzDanRer2NoY -enrichment
# refGene:cds 1.301%, blastzDanRer2NoY 3.980%, both 0.877%, cover 67.47%, 
# enrich 16.95x

# row count:
# danRer2  122160
# danRer2L8k 62815
# danRer2Default 75818
# danRer2NoY 124129
# can be pruned at the chaining step.
# trackDb - change Zebrafish Blastz to danRer1 Blastz and display this track
# for danRer2 as Zebrafish Blastz

# RESCORE DANRER2 BLASTZ (DONE, 2004-12-09, hartera)
    # Low scores can occur with repeats abridged and using the
    # HoxD55.q matrix. PSU's restore_rpts program rescored alignments
    # with the default matrix instead of the BLASTZ_Q matrix.
    # Rescore them here so the chainer sees the higher scores:
                                                                                
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.danRer2
    mkdir axtChrom.rescore
    foreach f (axtChrom/chr*.axt)
        axtRescore -scoreScheme=/cluster/data/blastz/HoxD55.q \
        $f axtChrom.rescore/$f:t
    end
    mv axtChrom axtChrom.orig
    mv axtChrom.rescore axtChrom
                                                                                
#   psl files and blastz tables will be the same regardless of score so
#   no need to reload

# CHAIN DANRER2 BLASTZ (DONE, 2004-12-09, hartera)
# RELOAD CHAINS WIH FILTERING (DONE, 2004-12-10, hartera)
# APPLY chainAntiRepeat TO REMOVE CHAINS THAT ARE THE RESULTS OF REPEATS
# AND DEGENERATE DNA (DONE, 2004-12-22, hartera)
    # Run axtChain on little cluster
    ssh kki
    cd /cluster/data/hg17/bed/blastz.danRer2
    mkdir -p axtChain/run1
    cd axtChain/run1
    mkdir out chain
    # create input list
    ls -1S /cluster/data/hg17/bed/blastz.danRer2/axtChrom/*.axt \
        > input.lst
    cat << '_EOF_' > gsub
#LOOP
doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy
                                                                                
    # Reuse gap penalties from hg16 vs chicken run.
    cat << '_EOF_' > ../../chickenHumanTuned.gap
tablesize^V     11
smallSize^V     111
position^V      1^V     2^V     3^V     11^V    111^V   2111^V  12111^V 32111^V
72111^V 152111^V        252111
qGap^V  325^V   360^V   400^V   450^V   600^V   1100^V  3600^V  7600^V  15600^V
31600^V 56600
bothGap^V       625^V   660^V   700^V   750^V   900^V   1400^V  4000^V  8000^V
16000^V 32000^V 57000
'_EOF_'
    # << this line makes emacs coloring happy
    cat << '_EOF_' > gsub
#LOOP
doChain {check in exists $(path1)} {check out line+ chain/$(root1).chain} {check out line+ out/$(root1).out}
#ENDLOOP
'_EOF_'
    # << this line makes emacs coloring happy
cat << '_EOF_' > doChain
#!/bin/csh
axtChain -scoreScheme=/cluster/data/blastz/HoxD55.q \
    -linearGap=../../chickenHumanTuned.gap $1 \
    /iscratch/i/gs.18/build35/bothMaskedNibs \ 
    /iscratch/i/danRer2/nib $2 >& $3
'_EOF_'
    # << this line makes emacs coloring happy
    chmod a+x doChain
    gensub2 input.lst single gsub jobList
    para create jobList
    para try, check, push, check...
# para time
# Completed: 45 of 46 jobs
# Crashed: 1 jobs
# CPU time in finished jobs:       1837s      30.62m     0.51h    0.02d  0.000 y
# IO & Wait Time:                   441s       7.35m     0.12h    0.01d  0.000 y
# Average job time:                  51s       0.84m     0.01h    0.00d
# Longest job:                      106s       1.77m     0.03h    0.00d
# Submission to last job:           419s       6.98m     0.12h    0.00d

    # crashed job is chr6_hla_hap1 which has no alignments
    # now on the cluster server, sort chains
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
    chainMergeSort run1/chain/*.chain > all.chain
    chainSplit chain all.chain
                                                                                
    # take a look at score distr's
    foreach f (chain/*.chain)
      grep chain $f | awk '{print $2;}' | sort -nr > /tmp/score.$f:t:r
      echo $f:t:r >> hist5000.out
      textHistogram -binSize=5000 /tmp/score.$f:t:r >> hist5000.out
      echo ""
    end

    # apart from chr19 not too many with chains with scores < 5000
    # load chr1 chain into table and check
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.danRer2/axtChain/chain
    hgLoadChain hg17 chr1_chainDanRer2 chr1.chain
# featureBits -chrom=chr1 hg17 refGene:cds chainDanRer2Link -enrichment
# refGene:cds 1.301%, chainDanRer2Link 3.676%, both 0.877%, cover 67.42%,
# enrich 18.34x
# featureBits -chrom=chr1 hg17 refGene:cds chainDanRer2 -enrichment
# refGene:cds 1.301%, chainDanRer2 32.611%, both 1.034%, cover 79.52%, 
# enrich 2.44x
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
    mv all.chain all.chain.unfiltered
    chainFilter -minScore=5000 all.chain.unfiltered > all.chain
    chainSplit chainFilt5k all.chain

    # load chr1 filtered chains and check
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.danRer2/axtChain/chainFilt5k
    hgLoadChain hg17 chr1_chainDanRer2Filt5k chr1.chain
# featureBits -chrom=chr1 hg17 refGene:cds chainDanRer2Filt5kLink -enrichment
# refGene:cds 1.301%, chainDanRer2Filt5kLink 2.907%, both 0.870%, cover 66.86%,
# enrich 23.00x
# featureBits -chrom=chr1 hg17 refGene:cds chainDanRer2Filt5k -enrichment
# refGene:cds 1.301%, chainDanRer2Filt5k 31.343%, both 1.028%, cover 79.02%, 
# enrich 2.52x
# checked browser - when filtered on minScore=5000, the low scoring 
# alignments removed are small and/or poor alignments so use this version.
    
    # remove repeats from filtered chains and reload into database
    # (2004-12-22, hartera)
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
    mv chainFilt5k chainRaw
    mkdir chain
    cd chainRaw
    foreach f (*.chain)
       set c = $f:r
       echo $c
       nice chainAntiRepeat /cluster/bluearc/hg17/bothMaskedNibs \
                       /cluster/bluearc/danRer2/nib $f \
                       ../chain/$c.chain
    end
    cd ..
    chainMergeSort ./chain/*.chain > all.chain.antirepeat
    chainSplit chainAR all.chain.antirepeat
    # load filtered chains and check
    ssh hgwdev
    echo 'drop table chr1_chainDanRer2Filt5k;' | hgsql hg17
    echo 'drop table chr1_chainDanRer2Filt5kLink;' | hgsql hg17
    # reload filtered chains instead of unfiltered (2004-12-10, hartera)
    # reload filtered chains with repeats removed (2004-12-22, hartera)
    cd /cluster/data/hg17/bed/blastz.danRer2/axtChain/
    cd chainAR
    foreach i (*.chain)
        set c = $i:r
        hgLoadChain hg17 ${c}_chainDanRer2 $i
        echo done $c
    end
# trackDb - change Zebrafish Chain to danRer1 Chain and display this track
# for danRer2 as Zebrafish Chain.
# after chainAntiRepeat
# featureBits -chrom=chr1 hg17 refGene:cds chainDanRer2Link -enrichment
# refGene:cds 1.304%, chainDanRer2Link 2.742%, both 0.872%, cover 66.81%, 
# enrich 24.36x 

# NET DANRER2 BLASTZ (DONE, 2004-12-09, hartera)
# RE-CREATE NET WITH FILTERED CHAINS (DONE, 2004-12-10, hartera)
# RE-DO NET WITH CHAINS FILTERED BY chainAntiRepeat (DONE, 2004-12-22, hartera) 
    ssh kksilo
    cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
    rm -r preNet
    mkdir preNet
    cd chainAR
    foreach i (*.chain)
       echo preNetting $i
       /cluster/bin/i386/chainPreNet $i ../../S1.len ../../S2.len \
                                     ../preNet/$i
    end
    cd ..
    mkdir n1
    cd preNet
    foreach i (*.chain)
      set n = $i:r.net
      echo primary netting $i
      /cluster/bin/i386/chainNet $i -minSpace=1 ../../S1.len ../../S2.len \
                                 ../n1/$n /dev/null
    end
    cd ..
    cat n1/*.net | /cluster/bin/i386/netSyntenic stdin noClass.net
    # memory usage 133443584, utime 905 s/100, stime 139
                                                                                
    # Add classification info using db tables:
    # netClass looks for ancient repeats in one of the databases
    # hg17 has this table - hand-curated by Arian but this is for
    # human-rodent comparisons so do not use here, use -noAr option
    mkdir -p /cluster/bluearc/danRer2/linSpecRep.notInHuman
    # linSpecRep.notInZebrafish exists for hg17 
    cp /iscratch/i/danRer2/linSpecRep.notInHuman/* \
       /cluster/bluearc/danRer2/linSpecRep.notInHuman
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
    time netClass noClass.net hg17 danRer2 zfishdanRer2.net \
         -tNewR=/cluster/bluearc/hg17/linSpecRep.notInZebrafish \
         -qNewR=/cluster/bluearc/danRer2/linSpecRep.notInHuman -noAr
    # 97.230u 54.290s 5:37.50 44.8%   0+0k 0+0io 217pf+0w
    # load net into database
    cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
    netFilter -minGap=10 zfishdanRer2.net |  hgLoadNet hg17 netDanRer2 stdin
# trackDb - change Zebrafish Net to danRer1 Net and display this track
# for danRer2 as Zebrafish Net.
# after chainAntiRepeat:
# featureBits hg17 refGene:cds netDanRer2 -enrichment
# refGene:cds 1.015%, netDanRer2 22.898%, both 0.783%, cover 77.15%, 
# enrich 3.37x

    # index had NULL cardinality, analyze table to fix (2005-1-18, Heather)
    hgsql hg17
    analyze table netDanRer2

# LOAD ACEMBLY TRACK (DONE, 2005-01-24, hartera)
# ACEMBLY TABLE RELOADED AND FINISHED COLOR CODING CODE IN 
# hgTracks (2005-01-28, hartera)
# FINISHED CODE FOR FILTERING BY GENE CLASS (2005-02-03, hartera)
    mkdir -p /cluster/data/hg17/bed/acembly
    cd /cluster/data/hg17/bed/acembly
    # Data is obtained from 
    # Danielle et Jean Thierry-Mieg     mieg@ncbi.nlm.nih.gov
 
    wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35.human.genes/acembly.ncbi_35.genes.proteins.fasta.tar.gz
    wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35.human.genes/acembly.ncbi_35.genes.gff.tar.gz
    wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35.human.genes/acembly.ncbi_35.mrnas.fasta.tar.gz
    wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35.human.genes/acembly.ncbi_35.genes.pfamhits.tar.gz
    tar xvzf acembly.ncbi_35.genes.gff.tar.gz
    tar xvzf acembly.ncbi_35.genes.proteins.fasta.tar.gz
    cd acembly.ncbi_35.genes.gff
    # the acembly dataset for hg16 had problems with reverse blocks so
    # check for these
cat << '_EOF_' > checkReversedBlocks
for i in x1*.gff
do
        echo -n "$i working ..."
        awk -F"\t" '
{
if ($4 > $5) {
    printf "reverse blocks problem for $1"
        printf "\n"
}
}
' $i > $i.fixed
        echo " done"
done
'_EOF_'
    # << this line makes emacs coloring happy
    chmod +x checkReversedBlocks
    ./checkReversedBlocks  
    ls -l *.fixed
    # all *.fixed files are empty so remove - there is no reversing of blocks
    rm *.fixed

    foreach f (x1.acemblygenes.*.gff)
      set c=$f:r:e
      egrep '^[a-zA-Z0-9]+\|NT_[0-9][0-9][0-9][0-9][0-9][0-9]' $f | \
        perl -wpe 's/^(\w+)\|(\w+)/$1\/$2/' > ctg-chr${c}_random.gff
      if (-e ../../../$c/lift/random.lft) then
        liftUp chr${c}_random.gff ../../../$c/lift/random.lft warn \
          ctg-chr${c}_random.gff
      endif
      grep -v ^$c\| $f | grep -v ^Hs | perl -wpe 's/^/chr/;' | \
        grep -v "^chr//" > chr$c.gff
      echo "done $c"
    end
    
    #- Load into database - use extended genePred
    ssh hgwdev
    cd /cluster/data/hg17/bed/acembly
    # Reloaded without -genePredExt 1/6/05:
    ldHgGene -gtf hg17 acembly acembly.ncbi_35.genes.gff/chr*.gff
    # for entry with 28212470 from chr6.gff, change to chr6 
    # and for 29124352 in chr6.gff, change to chr6 (1/13/05)
    echo 'update acembly set chrom = "chr6" where chrom = "chr28212470";' \
         | hgsql hg17 
    echo 'update acembly set chrom = "chr6" where chrom = "chr29124352";' \
         | hgsql hg17 
    
    # checkTableCoords and runGeneCheck to check data
    # a number of errors so leave on hgwdev for the moment
# checkTableCoords:
# rah.acembly has 16 records with chrom not described in chromInfo.
# rah.acembly item RPL10A.sNov04 chr6:35544172-35546520: end of last block (35546519) is not the same as chromEnd (35546520).
# rah.acembly has 1 records with blockEnd[n-1] != end.
# rah.acembly has 1 records with end > chromSize.
# chr6    acembly exon    35545934        35546101        .       +       0  
# gene_id RPL10A; transcript_id RPL10A.sNov04; exon_number 5
# chr6    acembly intron  35546102        35546520        .       +       0
# gene_id RPL10A; transcript_id RPL10A.sNov04; intron_type fuzzy
# chr6    acembly CDS     35546335        35546384        .       +       0
# gene_id RPL10A; transcript_id RPL10A.sNov04; exon_number 6
# chr6    acembly exon    35546335        35546519        .       +       0
# gene_id RPL10A; transcript_id RPL10A.sNov04; exon_number 6
# chr6    acembly stop_codon      35546382        35546384        .       +
# 0       gene_id RPL10A; transcript_id RPL10A.sNov04;
# here the intron overlaps exon 6 so take 35546519 to be txEnd
      echo 'update acembly set txEnd = 35546519 where name = "RPL10A.sNov04";'\
           | hgsql hg17
# for record where end > chromSize
      echo 'select * from acembly as a, chromInfo as c where c.chrom = a.chrom \            and c.size < a.txEnd;' | hgsql hg17
      # KIR2DL5.bNov04 on chr19_random, chr19_random size is 301858, 
      # txEnd is 305266 delete this record
      echo 'delete from acembly where name = "KIR2DL5.bNov04";' | hgsql hg17
# from runGeneCheck:
# 5780   inFrameStop
# 110664 noStart
# 23085 badCdsSplice
# 23848 noStop
# 14957 badUtrSplice
# 3661 gap
# 4726 badFrame
# 261066 lines in genePred.tab

    # e-mailed authors of data (2004-12-21, hartera) 
#  notiri.aNov04 - has ctg instead of atg at start. others have no start specified: sirora.nNov04
# sirora.zmNov04 - chr1:19389-19392 is AAC (gtt) (-)
# sirora.sNov04 - chr1:8925-8928 CAA (ttg) (-)
# sirora.rNov04 - chr1:8925-8928 CAA (ttg) (-)
    # for entries with 28212470 and 29124352 instead of chr6 change to chr6
    # Re-process this x1 file to chr6.gff (2005-01-24)
    mv x1.acemblygenes.6.gff x1.acemblygenes.6.gff.broken
    sed -e "s/^28212470/6/" x1.acemblygenes.6.gff.broken | sed -e \
           "s/^29124352/6/"  > x1.acemblygenes.6.gff
    grep -v ^6\| x1.acemblygenes.6.gff | grep -v ^Hs | perl -wpe 's/^/chr/;' | \
        grep -v "^chr//" > chr6.gff
    # Received a list of genes from Jean and Danielle Mieg
    # showing genes that are "main", "putative" or "cloud" - there should be
    # no "cloud" genes in our data set (2005-01-11)
    # download acembly_gene_lists.tar.gz from e-mail
    cd /cluster/data/hg17/bed/acembly
    tar xvzf acembly_gene_lists.tar.gz
    cd acembly_gene_lists
cat << '_EOF_' > getIDs.pl
#!/usr/bin/perl -w
use strict;

while (<STDIN>) {
   my @f = split(/\s+/);
   for (my $i =0; $i <= $#f; $i++) {
       if ($f[$i] =~ /gene_id$/) {
          # if field is ID type then next value is the ID
          my $id = $f[$i+1];
          # remove ";" at end and print ID
          chop $id;
          print "$id\n";
       }
   }
}
'_EOF_'
    chmod +x getIDs.pl
    # get gene IDs from gff files
    foreach f (../acembly.ncbi_35.genes.gff/chr*.gff)
       echo "Processing $f"
       perl getIDs.pl < $f >> genesGffs.ids
    end
    # remove back slash from some names
    sort genesGffs.ids | uniq > genesGffs.ids.uniq
    # reformat gene list to get just the genes and remove first 2 lines and sort
    foreach g (*.list)
       sed -e 's/"//g;' $g | sed -e 's/Gene : //;' | sed -e '1,2d' \
              | sort | uniq > $g.IDsort
    end 
    # remove back slash from some names
    perl -pi.bak -e 's/\\//' *.IDsort
    # check if cloud genes appear in gff files list of genes
    # list of genes in cloud but not in gff
    comm -13 genesGffs.ids.uniq cloud_gene.list.IDsort > gffvscloud.out
    diff gffvscloud.out cloud_gene.list.IDsort
    # there is no difference so none of the cloud genes are in the gff files
    # check if all the other genes in the main and putative lists are in gffs
    comm -13 genesGffs.ids.uniq main_gene.list.IDsort > gffvsmain.out
    comm -13 genesGffs.ids.uniq putative_gene.list.IDsort > gffvsputative.out
    wc -l *.out
    # 14 gffvsmain.out
    # 0 gffvsputative.out

    # there are 14 genes in the main set not in the gff files
    # actually there are 12, as FCA/MR and SLA/LP are in the gff files
    # all putative genes are in the gff set
    wc -l main_gene.list.IDsort putative_gene.list.IDsort
    # 52467 main_gene.list.IDsort
    # 43978 putative_gene.list.IDsort
    # 96445 total
    wc -l genesGffs.ids.uniq    
    # 97042 genesGffs.ids.uniq
    # check discrepancy
    cat main_gene.list.IDsort putative_gene.list.IDsort > mp.ids
    sort mp.ids > mp.sort
    comm -23 genesGffs.ids.uniq mp.sort > gffNotMP.out
    wc -l gffNotMP.out
    # 609 gffNotMP.out
# create table of Acembly gene classifications
# see http://www.ncbi.nlm.nih.gov/IEB/Research/Acembly/index.html?human
# in FAQ, describes main, putative and cloud genes. The cloud genes are not 
# well confirmed and so they are not in this data set. 
    # NEED TO FILTER GENES AND RELOAD TABLES:
    # authors Jean and Danielle Mieg e-mailed back. The 12 genes in the 
    # putative list that are not in the gff files were not exported
    # as they did not find a single putative protein to describe so they 
    # were not added to the gffs. They will be added at a later date.
    # Remove these from the acemblyClass table (2005-01-21, hartera)
    # Reload acemblyClass table as problems with the gene names
    # the class table has gene IDs and the acembly table has transcript IDs
    # it is hard to look up class in the class table since just removing the 
    # transcript ID suffixes (e.g. "aNov04" after a ".") does not work as 
    # some gene IDs have a "." in them anyway.

    ssh kksilo
    cd /cluster/data/hg17/bed/acembly/acembly_gene_lists
    comm -13 gffvsmain.out main_gene.list.IDsort > main_gene.list.filt
    wc -l main_gene.list.filt
    # 52455 main_gene.list.filt
    ssh hgwdev
    cd /cluster/data/hg17/bed/acembly/acembly_gene_lists
    # drop acemblyClass table and recreate (2005-01-27, hartera)
    echo 'drop table acemblyClass;' | hgsql hg17
    # prepare a file of genes and classification
    # use transcript IDs - get these and corresponding gene IDs from gff files
    # if Gene IDs used the difficult to parse transcript ID (name column) from 
    # the acembly genePred table. e.g. notiri.aNov04 is a transcript ID so can
    # remove suffix after "." to obtain gene ID, some gene names have "." in
    # them and not all have the suffix.
    # 260446 transcript IDs (use allFiltered.gff - see below) 
    perl getClass.pl main_gene.list.filt putative_gene.list.IDsort \
         ../acembly.ncbi_35.genes.gff/allFiltered.gff 
    foreach f (main_gene.list.filt putative_gene.list.IDsort)
       if ($f == "main_gene.list.filt") then
          set t = "main"
       endif
       if ($f == "putative_gene.list.IDsort") then
          set t = "putative"
       endif
       awk 'BEGIN {OFS="\t"} {print $1, "'$t'"}' $f >> class.txt
    end
    sort classes.txt | uniq > geneIDtxID.class
    # get transcript ID and class fields for acemblyClass table
    awk 'BEGIN {OFS="\t"} {print $2,$3}' geneIDtxID.class > acemblyClass.tab
    wc -l acemblyClass.tab
    # 260446 acemblyClass.tab
    # make change to acemblyClass.as and check in:
    # change name to be transcript ID instead of gene ID
cat << '_EOF_' > $HOME/kent/src/hg/lib/acemblyClass.as
table acemblyClass
"Class for Acembly genes"
   (
    string name; "Transcript ID for Acembly gene"
    string class; "Class of gene"   
   )
'_EOF_'
    cd $HOME/kent/src/hg/lib/
    autoSql acemblyClass.as acemblyClass
    mv acemblyClass.h $HOME/kent/src/hg/inc
    # do make to check it works and commit the .as, .sql, .c and .h files to CVS
    cd /cluster/data/hg17/bed/acembly/acembly_gene_lists
    echo "drop table acemblyClass" | hgsql hg17
    hgsql hg17 < ~/kent/src/hg/lib/acemblyClass.sql
    # reload table with transcript IDs 
    echo "load data local infile 'acemblyClass.tab' into table acemblyClass" \
         | hgsql hg17
    
    # There were also 609 genes in the gff files that are not in the
    # main, putative or cloud gene lists. Jean and Danielle Mieg say that
    # these were filtered out from their data set but not from the gff files.
    # Remove these from the gff files. (gffNotMP.out) (2005-01-24)
    cd /cluster/data/hg17/bed/acembly/acembly.ncbi_35.genes.gff
    cat chr*.gff > all.gff
cat << '_EOF_' > removeGenes.pl
#!/usr/bin/perl -w
use strict;
                                                                                
my $genes = $ARGV[0];
my $gff = $ARGV[1];
                                                                                
open(GENES, $genes) || die "Can not open $genes:$!\n";
open(GFF, $gff) || die "Can not open $gff:$!\n";
open(OUT, ">removed.out") || die "Can not open removed.out:$!\n";
                                                                                
my %genes;
                                                                                
while (<GENES>) {
   chomp;
   my $g = $_;
   $genes{$g} = 1;
}
close GENES;
                                                                                
while (<GFF>) {
   my $l = $_;
   my $id;
   my @line = split(/\s+/);
   for (my $i = 0; $i <= $#line; $i++) {
      if ($line[$i] eq "gene_id") {
         $id = $line[$i+1];
      }
   }
   $id =~ s/;//;
   print "id is now $id\n";
   if (!exists($genes{$id})) {
      print $l;
   }
   else {
      print OUT $l;
   }
}
'_EOF_'
    perl removeGenes.pl ../acembly_gene_lists/gffNotMP.out all.gff \
                        > allFiltered.gff 
    # checked that gene IDs in the removed.out file are the same
    # same as those in gffNotMP.out
    # reload into the acembly table
    ssh hgwdev
    cd /cluster/data/hg17/bed/acembly
    echo 'drop table acembly;' | hgsql hg17
    # Reloaded with filtered set 2005-01-23, reload again 2005-01-28 with
    # the genePredExt option to get gene ID in name 2 field
    ldHgGene -gtf -genePredExt hg17 acembly \
             acembly.ncbi_35.genes.gff/allFiltered.gff
    # Read 260446 transcripts in 3656676 lines in 1 files
    #  260446 groups 41 seqs 1 sources 5 feature types
    #  260446 gene predictions
    # remove cdsStartStat, cdsEndStat and exonFrames fields
    echo 'alter table acembly drop column cdsStartStat;' | hgsql hg17
    echo 'alter table acembly drop column cdsEndStat;' | hgsql hg17
    echo 'alter table acembly drop column exonFrames;' | hgsql hg17

    # fix problem data found by checkTableCoords
    # here the intron overlaps exon 6 so take 35546519 to be txEnd
    echo 'update acembly set txEnd = 35546519 where name = "RPL10A.sNov04";'\
           | hgsql hg17
    # for record where end > chromSize
    echo 'select * from acembly as a, chromInfo as c where c.chrom = a.chrom \            and c.size < a.txEnd;' | hgsql hg17
    # KIR2DL5.bNov04 on chr19_random, size is 301858, txEnd is 305266
    # delete this record
    echo 'delete from acembly where name = "KIR2DL5.bNov04";' | hgsql hg17
    # acembly peptide table
    # need to just grab same sequences that are in acembly
    cd ./acembly.ncbi_35.genes.proteins.fasta
    echo 'select name from acembly;' | hgsql -N hg17 > acembly.name
    cat *.fasta > allPep.fa
    faSomeRecords allPep.fa acembly.name acemblyPep.fa
    # PEPTIDE SEQUENCES NOT LOADED
    # There are 236,554 peptide names that do not match transcript IDs in
    # the acembly table and 110,278 transcript IDs in acembly that do not
    # have a corresponding peptide. Waiting for repsonse about this from
    # Jean and Danielle (2005-01-31)  
    # hgPepPred hg17 generic acemblyPep \
    #         acembly.ncbi_35.genes.proteins.fasta/*.fasta

    # Edit hgTracks.c to get colour coded tracks based on the gene class
    # for each gene as read from the acemblyClass table.
    # Edits to hui.c, hgTrackUi.c and hgTracks.c to allow filtering of 
    # genes based on class.
# acembly trackDb entry:
# track acembly
# shortLabel Acembly Genes
# longLabel AceView Gene Models With Alt-Splicing
# group genes
# priority 41
# visibility dense
# color 155,0,125
# type genePred acemblyPep acemblyMrna
# url http://www.ncbi.nih.gov/IEB/Research/Acembly/av.cgi?db=hg17&l=$$
# itemClassTbl acemblyClass
# geneClasses main putative
# gClass_main 128,0,125
# gClass_putative 200,0,125
# urlLabel Transcript ID:

# search added:
# searchTable acembly
# searchType genePred
# searchMethod prefix
# termRegex [^[:space:]]+
# searchPriority 50

    # Received data with gene product relationship from Jean Thierry-Mieg
    # (2005-02-17)
    ssh eieio
    cd /cluster/data/hg17/bed/acembly/acembly.ncbi_35.genes.proteins.fasta
    wget --timestamp ftp://ftp.ncbi.nlm.nih.gov/repository/acedb/human/acembly.ncbi_35.gene2product.txt.gz
    gunzip acembly.ncbi_35.gene2product.txt.gz
    # these are gene ID and product mappings, need transcript ID to product
    # mappings. E-mailed Jean Thierry-Mieg to ask for this information
    
# BUILD WGRNA TRACK (DONE, 2004-12-13, Fan)

# Grab data from original miRNA track and convert them into wgRna .tab format.
    hgsql hg17 --skip-column-names -e 'select * from miRNA' >miRNA.out
    cat miRNA.out  | awk {'print $2"\t"$3"\t"$4"\t"$5"\t"$6"\t"$7"\t"$8"\t"$9"\t""miRna"'} >wgRna.tab

# Break the original custom track data file, hsa-snoRNA_track.txt, into two files j1 and j2, 
# then remove header and blank lines.
    cat j1 | awk {'print $1"\t"$2"\t"$3"\t"$4"\t"$5"\t"$6"\t"0"\t"0"\t""CDBox"'} >>wgRna.tab
    cat j2 | awk {'print $1"\t"$2"\t"$3"\t"$4"\t"$5"\t"$6"\t"0"\t"0"\t""HAcaBox"'} >>wgRna.tab

# load into wgRna table
    hgLoadBed -sqlTable=/cluster/home/fanhsu/hg/lib/wgRna.sql hg17 wgRna wgRna.tab
# create and edit wgRna.html under src/hg/makeDb/trackDb/human/hg17.

# RELOADED wgRna DATA USING wgRNA_corrected.txt SENT BY MICHEL WEBER
# Manually removed the first header line and the first column of the bin field and removed 
# the last empty line.

    cut -f 2- wgRNA_corrected.txt >wgRna.tab
    vi wgRna.tab
    hgLoadBed -sqlTable=/cluster/home/fanhsu/hg/lib/wgRna.sql hg17 wgRna wgRna.tab


# UPDATED WGRNA DATA PER EMAIL FROM WEBER (2004-12-14, Fan).

# Added the following 3 lines to j1
chr3 161715396 161715726 U90 480 -
chr11 93104316 93104387 Z40 480 -
chr11 93106041 93106114 mgh28S-2410 480 -
    
# Regenerated wgRna table
    cat miRNA.out  | awk {'print $2"\t"$3"\t"$4"\t"$5"\t"$6"\t"$7"\t"$8"\t"$9"\t""miRna"'} >wgRna.tab
    cat j1 | awk {'print $1"\t"$2"\t"$3"\t"$4"\t"$5"\t"$6"\t"0"\t"0"\t""CDBox"'} >>wgRna.tab
    cat j2 | awk {'print $1"\t"$2"\t"$3"\t"$4"\t"$5"\t"$6"\t"0"\t"0"\t""HAcaBox"'} >>wgRna.tab
    hgLoadBed -sqlTable=/cluster/home/fanhsu/hg/lib/wgRna.sql hg17 wgRna wgRna.tab

# Changed the following records to RNA type scaRna.

hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U85"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U87"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U88"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U89"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U90"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U91"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U92"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U93"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U100"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA26"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA35"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA45"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA47"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA57"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="HBII-382"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="mgU2-19/30"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="mgU2-25/61"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="mgU2-22/U4-8"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U85"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U87"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U88"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U89"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U90"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U91"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U92"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U93"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="U100"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA26"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA35"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA45"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA47"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="ACA57"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="HBII-382"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="mgU2-19/30"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="mgU2-25/61"'
hgsql hg17 -e 'update wgRna set type = "scaRna" where name ="mgU2-22/U4-8"'

# Updated .../trackDb/human/hg17/wgRna.html.

# MAKE VSDANRER2 DOWNLOADABLES (DONE, 2004-12-14, hartera)
# REMAKE FOR CHAINS AND NET AFTER USING chainAntiRepeat
# (DONE, 2004-12-22, hartera)
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.danRer2/axtChrom
    set gp = /usr/local/apache/htdocs/goldenPath/hg17
    mkdir -p $gp/vsDanRer2/axtChrom
    cp -p *.axt $gp/vsDanRer2/axtChrom
    cd $gp/vsDanRer2/axtChrom
    gzip *.axt
    md5sum *.gz > md5sum.txt
                                                                                
    # copy chains and nets to downloads area
    # re-make chains and net downloadables (2004-12-22, hartera)
    rm $gp/vsDanRer2/zebrafish*.gz $gp/vsDanRer2/md5sum.txt
    cd /cluster/data/hg17/bed/blastz.danRer2/axtChain
    gzip -c all.chain.antirepeat > \
            /cluster/data/hg17/zip/zebrafishDanRer2.chain.gz
    gzip -c zfishdanRer2.net > /cluster/data/hg17/zip/zebrafishDanRer2.net.gz
    cd $gp/vsDanRer2
    mv /cluster/data/hg17/zip/zebrafish*.gz .
    md5sum *.gz > md5sum.txt
    # Copy over & edit README.txt w/pointers to chain, net formats.
                                                                                
# CLEANUP DANRER2 BLASTZ (DONE, 2004-12-14, hartera)
# RE-DONE (DONE, 2004-12-22, hartera)
# REMOVED RAW AND LAV DIRS (DONE, 2005-02-24, hartera)
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.danRer2
    nice rm axtChain/run1/chain/* &
    nice rm -fr axtChain/n1 axtChain/noClass.net &
    nice gzip axtChrom/* pslChrom/* axtChain/all.chain axtChain/all.chain.unfiltered axtChain/*.net &
    nice gzip axtChain/all.chain.antirepeat axtChain/chainAR/*.chain &
    nice rm -fr axtChain/chain axtChain/chainRaw axtChain/preNet &
    nice rm -rf raw &
    nice rm -rf lav &

# EXTRACT AXT'S AND MAF'S FROM TETRAODON (tetNig1) NET 
# (DONE, 2004-12-15, hartera) 
# Redo to remove overlaps (2006-04-07 kate)
    ssh eieio
    # create axts
    cd /cluster/data/hg17/bed/blastz.tetNig1/axtChain
    netSplit tetNig1.net tetraodonNet
    mkdir -p ../axtNet
cat > axtNet.csh << 'EOF'
    foreach f (tetraodonNet/chr*.net)
        set c = $f:t:r
        echo "axtNet on $c"
        netToAxt tetraodonNet/$c.net chain/$c.chain \
        /cluster/data/hg17/nib /cluster/data/tetNig1/nib ../axtNet/$c.axt
        echo "Complete: $c.net -> $c.axt"
        end
'EOF'

    chmod +x axtNet.csh
    csh axtNet.csh >&! axtNet.log &
    tail -100f axtNet.log

    # sort axts before making mafs - must be sorted for multiz
    cd /cluster/data/hg17/bed/blastz.tetNig1
    mv axtNet axtNet.unsorted
    mkdir axtNet
    foreach f (axtNet.unsorted/*.axt)
        set c = $f:t:r
        echo "Sorting $c"
        axtSort $f axtNet/$c.axt
    end

    # create maf
    ssh eieio
    cd /cluster/data/hg17/bed/blastz.tetNig1
    cd axtNet
    mkdir ../mafNet

cat > makeMaf.csh << 'EOF'
    foreach f (chr*.axt)
      set maf = $f:t:r.tetNig1.maf
      echo translating $f to $maf
      axtToMaf $f \
            /cluster/data/hg17/chrom.sizes /cluster/data/tetNig1/chrom.sizes \
            ../mafNet/$maf -tPrefix=hg17. -qPrefix=tetNig1.
    end
'EOF'
    csh makeMaf.csh >&! makeMaf.log &
    tail -100f makeMaf.log

    nice gzip axtChrom/* pslChrom/* axtChain/all.chain axtChain/*.net &

    # redo axt's and maf's to remove overlaps (2006-04-07 kate)
    ssh kkstore02
    cd /cluster/data/hg17/bed/blastz.tetNig1
    mv axtNet axtNet.old
    mv mafNet mafNet.old
    mkdir -p axtNet mafNet
    cd axtChain

cat > fix.csh << 'EOF'
    date
    foreach f (tetraodonNet/chr*.net)
        set c = $f:t:r
        echo $c
        netToAxt tetraodonNet/$c.net chain/$c.chain \
            /cluster/data/hg17/nib /cluster/data/tetNig1/nib stdout | \
                axtSort stdin ../axtNet/$c.axt
        echo "Complete: $c.net -> $c.axt"
        axtToMaf ../axtNet/$c.axt \
            /cluster/data/hg17/chrom.sizes /cluster/data/tetNig1/chrom.sizes \
            ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=tetNig1.
    end
    date
'EOF'
    csh fix.csh >&! fix.log &
    cd /san/sanvol1/scratch/hg17/mafNet
    rm -fr tetNig1
    cp -rp /cluster/data/hg17/bed/blastz.tetNig1/mafNet tetNig1


# 10-WAY MULTIZ -- 8-WAY PLUS FROG AND TETRA (DONE 2004-12-22 kate)
#       Use older multiz (not v10) till bugs fixed

    ssh eieio
    cd /cluster/data/hg17/bed
    rm multiz10way
    mkdir multiz.2004-12-22
    ln -s multiz.2004-12-22 multiz10way
    cd multiz10way

    cat > tree.nh << 'EOF'
((((((hg17,panTro1),(rn3,mm5)),canFam1),galGal2),xenTro1),((fr1,tetNig1),danRer1))
'EOF'

    mkdir /cluster/bluearc/hg17/multiz.2004-12-22
    cd /cluster/bluearc/hg17
    mkdir 2004-12-22
    rm multiz10way
    ln -s multiz.2004-12-17 multiz10way.v10
    ln -s multiz.2004-12-22 multiz10way
    # reuse pairwise MAF's on bluearc
    mv multiz10way.v10/{canFam1,danRer1,fr1,galGal2,mm5,panTro1,rn3,tetNig1,xenTro1} multiz10way
    # NOTE: pairwise mafs were moved to /cluster/bluearc/hg17/mafNet

    # make output dir and run dir
    ssh kk9
    cd /cluster/data/hg17/bed
    cd multiz10way
    mkdir -p maf
    mkdir -p run
    cd run

    # create scripts to run multiz on cluster
cat > oneMultiz.csh << 'EOF'
#!/bin/csh -fe
    set c = $1
    set multi = /scratch/$user/multiz10way.$c
    set pairs = /cluster/bluearc/hg17/multiz10way

    # special mode --
    # with 1 arg, cleanup
    if ($#argv == 1) then
        rm -fr $multi
        exit
    endif

    set s1 = $2
    set s2 = $3

    # locate input files -- in pairwise dir, or multiple dir
    set d1 = $multi
    set d2 = $multi
    if (-d $pairs/$s1) then
        set d1 = $pairs
    endif
    if (-d $pairs/$s2) then
        set d2 = $pairs
    endif
    set f1 = $d1/$s1/$c.maf
    set f2 = $d2/$s2/$c.maf
    # write to output dir
    set out = $multi/${s2}${s1}
    mkdir -p $out

    # check for empty input file
    if (-s $f1 && -s $f2) then
        echo "Aligning $f1 $f2"
        /cluster/bin/penn/multiz $f1 $f2 - > $out/$c.maf
    else if (-s $f1) then
        cp $f1 $out
    else if (-s $f2) then
        cp $f2 $out
    endif
'EOF'
# << for emacs
chmod +x oneMultiz.csh

cat > allMultiz.csh << 'EOF'
#!/bin/csh -fe

set c = $1
oneMultiz.csh $c mm5 panTro1
oneMultiz.csh $c rn3 panTro1mm5 
oneMultiz.csh $c canFam1 panTro1mm5rn3 
oneMultiz.csh $c galGal2 panTro1mm5rn3canFam1
oneMultiz.csh $c xenTro1 panTro1mm5rn3canFam1galGal2
oneMultiz.csh $c fr1 panTro1mm5rn3canFam1galGal2xenTro1
oneMultiz.csh $c tetNig1 panTro1mm5rn3canFam1galGal2xenTro1fr1
oneMultiz.csh $c danRer1 panTro1mm5rn3canFam1galGal2xenTro1fr1tetNig1
# get final alignment file
cp /scratch/$user/multiz10way.$c/panTro1mm5rn3canFam1galGal2xenTro1fr1tetNig1danRer1/$c.maf /cluster/data/hg17/bed/multiz10way/maf/$c.maf
#cleanup
oneMultiz.csh $c
'EOF'
# << for emacs
chmod +x allMultiz.csh

cat > gsub << 'EOF'
#LOOP
allMultiz.csh $(root1) {check out line+ /cluster/data/hg17/bed/multiz10way/maf/$(root1).maf}
#ENDLOOP
'EOF'
# << for emacs
    cut -f 1 /cluster/data/hg17/chrom.sizes > chrom.lst
    gensub2 chrom.lst single gsub jobList
    para create jobList
    para try; para check
    para push

    # post-process multiz maf with maf_project to "glue" short
    # alignment blocks together
    ssh eieio
    cd /cluster/data/hg17/bed/multiz10way.v8
    mkdir -p mafGlued
    cd maf
    foreach f (*.maf)
        set c = $f:r
        echo "gluing $f"
        /cluster/bin/penn/maf_project $f hg17.$c > ../mafGlued/$c.maf
    end
    # filter out alignment blocks with no alignments in non-reference species,
    # and low-scoring alignments based on Webb Miller's latest
    # recommendations (score < -5 * ncol^2 * nrow)
    # NOTE: Webb hasn't approved the filtered alignments yet,
    # so leaving them in for now.
    #mkdir -p mafFiltered
    #cd ../mafGlued
    #foreach f (*.maf)
        #set c = $f:r
        #echo "filtering $f"
        #~kate/bin/i386/mafFilter -factor $f > ../mafFiltered/$c.maf
    #end
    #cd ..

    grep score mafGlued/chr1.maf | wc -l
    grep score mafFiltered/chr1.maf | wc -l
    grep score mafGlued/bad | wc -l
        # 43692
    grep score=0.0 bad | wc -l
        # 10206
    # load alignments into tables
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz10way.v8
    set mafDir = /gbdb/hg17/mafNet
    mkdir -p $mafDir

    # multiple alignment
    set mafDir = /gbdb/hg17/multiz10way/maf
    mkdir -p $mafDir/multiz10way
    cd /cluster/data/hg17/bed/multiz10way.v8/mafGlued
    ln -s `pwd`/*.maf $mafDir/multiz10way
    hgLoadMaf hg17 -warn multiz10way -pathPrefix=$mafDir/multiz10way

    # load summary table to replace pairwise
    cd /cluster/data/hg17/bed/multiz10way.v8/mafGlued/
    time cat chr*.maf | hgLoadMafSummary -minSize=10000 -mergeGap=500 -maxSize=50000 hg17 multiz10waySummary stdin
    # Processed 27314693 components in 9081437 mafs from stdin
    # 30 minutes


# CONSERVATION SCORING WITH PHASTCONS (DONE 2005-01-14 kate)
# 1. Partition multiple alignment into windows, using "msa_split"
# 2. Create starting tree model, with branch lengths
#       use "phyloFit" on alignments
# 3. Estimate GC avg. over all species, use "msa_view" on maf
# 4. Estimate other model params, using phastCons (via doEstimate script)

    # NOTE: no alignment filtering done -- the scores don't look
    # particularly meaningful w/ this version of multiz.
    # Next time, run on "glued" (maf_projected) 

    ssh eieio
    cd /cluster/data/hg17/bed/multiz10way.v8
    set mafDir = /cluster/bluearc/hg17/multiz10way.v8/maf
    mkdir -p $mafDir
    cp -r maf/*.maf $mafDir

    ssh kk9
    cd /cluster/data/hg17/bed/multiz10way.v8
    mkdir cons
    cd cons

    # break up the genome-wide MAFs into pieces
    # NOTE: chrom fasta files are already on the bluearc
    # from previous run
    mkdir /cluster/bluearc/hg17/chrom
    cd /cluster/data/hg17
    foreach f (`cat chrom.lst`)
        echo $f
        cp -r $f/*.fa /cluster/bluearc/hg17/chrom
    end

    cd /cluster/data/hg17/bed/multiz10way.v8/cons
    mkdir run.split
    cd run.split
    set WINDOWS = /cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS
    rm -fr $WINDOWS
    mkdir -p $WINDOWS
    cat << 'EOF' > doSplit.sh
#!/bin/sh

PHAST=/cluster/bin/phast
FA_SRC=/cluster/bluearc/hg17/chrom
WINDOWS=/cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS

maf=$1
c=`basename $maf .maf`
echo $c
mkdir -p /scratch/msa_split
${PHAST}/msa_split $maf -i MAF -M ${FA_SRC}/$c.fa -O hg17,panTro1,mm5,rn3,canFam1,galGal2,xenTro1,fr1,tetNig1,danRer1 -w 1000000,0 -r /scratch/msa_split/$c -o SS -I 1000 -B 5000
[ $? -eq 0 ] || exit 1
echo "Copying..."
cd /scratch/msa_split
for file in $c.*.ss ; do gzip -c $file > ${WINDOWS}/$file.gz ; done
[ $? -eq 0 ] || exit 1
rm -f /scratch/msa_split/$c.*.ss
echo "Done copying"
echo "Done" >> ${WINDOWS}/$c.done
'EOF'
# << for emacs

    set WINDOWS = /cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS
    chmod +x doSplit.sh
    rm -f jobList
    foreach file (/cluster/bluearc/hg17/multiz10way.v8/maf/*.maf) 
        set c = $file:t:r
	echo "doSplit.sh $file {check out line+ $WINDOWS/$c.done}" >> jobList
    end
    
    para create jobList
        # 46 jobs
    para try
    para check

    # TODO: cleanup
    # rm -fr $mafDir
    
    # now generate conservation scores and predicted elements
    set path = ($path /cluster/bin/phast); rehash
    cd /cluster/data/hg17/bed/multiz10way.v8/cons
    mkdir run.elements   
    cd run.elements

    # create a starting tree model from a chr1 ss files in WINDOWS dir.
    ssh kolossus
    cd /cluster/data/hg17/bed/multiz10way.v8/cons/run.elements
    gunzip -c /cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS/chr1.14996059-15998256.ss.gz \
                > /tmp/phastCons.$$
     phyloFit -i SS /tmp/phastCons.$$ --out-root starting-tree --tree \
        "((((((hg17,panTro1),(mm5,rn3)),canFam1),galGal2),xenTro1),((fr1,tetNig1),danRer1))" 
    rm /tmp/phastCons.$$
    cat starting-tree.mod
#ALPHABET: A C G T 
#ORDER: 0
#SUBST_MOD: REV
#TRAINING_LNL: -2635749.517410
#BACKGROUND: 0.247225 0.248374 0.250827 0.253574 
#RATE_MAT:
#-0.997890    0.201447    0.648573    0.147870 
#0.200515   -1.020796    0.190184    0.630096 
#0.639258    0.188324   -1.025170    0.197587 
#0.144168    0.617176    0.195447   -0.956791 
#TREE: ((((((hg17:0.006401,panTro1:0.008342):0.099376,(mm5:0.083404,rn3:0.105411):0.242694):0.020883,canFam1:0.221922):0.099131,galGal2:0.275759):0.041997,xenTro1:0.280306):0.064815,((fr1:0.137674,tetNig1:0.091463):0.118573,danRer1:0.250847):0.064815);

    # estimate model parameters
    # estimate avg. cross-species avg. GC content from chr1 maf's
    ssh kolossus
    set path = ($path /cluster/bin/phast); rehash
    cd /cluster/data/hg17/bed/multiz10way.v8/cons/run.elements
    msa_view --aggregate hg17,panTro1,rn3,mm5,canFam1,galGal2,xenTro1,danRer1,tetNig1,fr1 \
        -i MAF \
        --summary-only /cluster/data/hg17/bed/multiz10way.v8/maf/chr1.maf\
            > maf_summary.txt
    awk '$1 == "[aggregate]" {printf "%0.3f\n", $3 + $4}' maf_summary.txt
       # 0.424

    # generate models from random sample of genome (use 90 1Mb windows,
    # to conveniently run on rack 9 100-node cluster)
    # On first pass, used parameters from 8way alignment:
    #   expected-lengths 12 -taret-coverage .17
    # NOTE: there may be a cleverer way to select the first length param
    # On second pass, used parameters below, based on consEntropy
    # and featureBits coverage of elements, below
    cat << 'EOF' > doEstimate.sh
#!/bin/sh
zcat $1 | /cluster/bin/phast/phastCons - starting-tree.mod --gc 0.424 --nrates 1,1 --no-post-probs --ignore-missing --expected-lengths 11 --target-coverage 0.20 --quiet --log $2 --estimate-trees $3
'EOF'
    chmod u+x doEstimate.sh
    rm -fr LOG TREES
    mkdir -p LOG TREES
    ls /cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS/*.gz > all.windows
    /cluster/bin/phast/chooseLines -k 90 all.windows > subset.windows
    rm -f jobs.lst
    foreach f (`cat subset.windows`)
        set root = $f:t:r:r
	echo doEstimate.sh /cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS/$f LOG/$root.log TREES/$root >> jobs.lst
    end
    # run cluster job (about an hour)
    ssh kk9
    cd /cluster/data/hg17/bed/multiz10way.v8/cons/run.elements
    para create jobs.lst
       # 90 jobs written to batch
    para try; para check
    para push
# 2 jobs crashed with out-of-mem; as we are just taking a sample
# this is probably OK, but I've notified Adam
# Average job time:                1055s      17.58m     0.29h    0.01d
# Longest job:                     3647s      60.78m     1.01h    0.04d

    # NOTE: should have used ave.noncons.mod to improve parameter estimation
    # cp nave.noncons.mod starting-tree.mod

    ls TREES/*.cons.mod > cons.txt
    /cluster/bin/phast/phyloBoot --read-mods '*cons.txt' --output-average ave.cons.mod > cons_summary.txt
    grep TREE ave.cons.mod
# TREE: ((((((hg17:0.002313,panTro1:0.002931):0.036375,(mm5:0.029849,rn3:0.039008):0.095334):0.003258,canFam1:0.078205):0.047189,galGal2:0.158045):0.020103,xenTro1:0.169387):0.028857,((fr1:0.071610,tetNig1:0.057766):0.091165,danRer1:0.138905):0.028857);
    ls TREES/*.noncons.mod > noncons.txt
    /cluster/bin/phast/phyloBoot --read-mods '*noncons.txt' --output-average ave.noncons.mod > noncons_summary.txt
    grep TREE ave.noncons.mod
# TREE: ((((((hg17:0.007342,panTro1:0.009340):0.116009,(mm5:0.095037,rn3:0.124288):0.304355):0.010633,canFam1:0.249367):0.151476,galGal2:0.507037):0.064317,xenTro1:0.549121):0.094733,((fr1:0.231246,tetNig1:0.185161):0.296288,danRer1:0.446734):0.094733);


    # analyze conservation genome-wide
    cat << 'EOF' > doPhastCons.sh
#!/bin/sh

mkdir -p /cluster/bluearc/hg17/multiz10way.v8/phastCons/POSTPROBS /cluster/bluearc/hg17/multiz10way.v8/phastCons/ELEMENTS
pref=`basename $1 .ss.gz`
chr=`echo $pref | awk -F\. '{print $1}'`
tmpfile=/scratch/phastCons.$$
zcat $1 | /cluster/bin/phast/phastCons - ave.cons.mod,ave.noncons.mod --expected-lengths 11 --target-coverage 0.20 --quiet --seqname $chr --idpref $pref --viterbi /cluster/bluearc/hg17/multiz10way.v8/phastCons/ELEMENTS/$pref.bed --score --require-informative 0 > $tmpfile
gzip -c $tmpfile > /cluster/bluearc/hg17/multiz10way.v8/phastCons/POSTPROBS/$pref.pp.gz
rm $tmpfile
'EOF'
    chmod u+x doPhastCons.sh

    rm -fr /cluster/bluearc/hg17/multiz10way.v8/phastCons/POSTPROBS /cluster/bluearc/hg17/multiz10way.v8/phastCons/ELEMENTS
    rm -f jobs2.lst
    foreach f (/cluster/bluearc/hg17/multiz10way.v8/cons/WINDOWS/*.ss.gz)
        echo doPhastCons.sh $f >> jobs2.lst
    end

    # run cluster job (it's quick -- 10 minutes or so)
    ssh kk
    cd /cluster/data/hg17/bed/multiz10way.v8/cons/run.elements
    para create jobs2.lst
        # 2932 jobs written to batch
    para try; para check
    para push
# Average job time:                  80s       1.33m     0.02h    0.00d
# Longest job:                      157s       2.62m     0.04h    0.00d
# Submission to last job:           583s       9.72m     0.16h    0.01d

    # combine predictions and transform scores to be in 0-1000 interval
    # do in a way that avoids limits on numbers of args
    rm -f splitfiles* all.raw.bed
    find /cluster/bluearc/hg17/multiz10way.v8/phastCons/ELEMENTS -name "*.bed" > files
    split files splitfiles
    foreach s (splitfiles*)
       awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", $1, $2, $3, $5, $5;}' `cat $s` >> all.raw.bed
    end
    /cluster/bin/scripts/lodToBedScore all.raw.bed > all.bed
    rm files splitfiles* 

    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz10way.v8/cons/run.elements
    hgLoadBed hg17 phastConsElements10way all.bed
    sort -rn -k 5 all.bed | sed -n '1,100000p' > top100K.bed
    hgLoadBed hg17 phastConsElements10wayTop100K top100K.bed

    # check coverage -- reran estimation and conservation steps with new parameters till
    # coverage close to 5% and expected-length parameter is close to consEntropy recommended length
    
    featureBits hg17 phastConsElements10way
        # first pass
        #       .17 12
        # 132657993 bases of 2866216770 (4.628%) in intersection
        # second pass -- used this
        #       .20 11
        # 143386170 bases of 2866216770 (5.003%) in intersection
    featureBits hg17 phastConsElements
        # 137850739 bases of 2866216770 (4.810%) in intersection

    # check expected-length parameter
    # first pass
    /cluster/bin/phast/consEntropy .17 12 \
                        ave.cons.mod ave.noncons.mod --NH 9.78
        # recommended length 10.4
    # second pass -- good enough according to Adam
    /cluster/bin/phast/consEntropy .20 11 \
                        ave.cons.mod ave.noncons.mod --NH 9.78
        #( Solving for new omega: 11.000000 12.243251 12.155776 12.155369 )
        #Transition parameters: gamma=0.200000, omega=11.000000, mu=0.090909, nu=0.022727
        #Relative entropy: H=1.263205 bits/site
        #Required length: N=7.548911 sites
        #Total entropy: NH=9.535821 bits
        #Recommended expected length: omega=12.155369 sites (for NH=9.780000)
 
    # create wiggle data files
    ssh eieio
    cd /cluster/data/hg17/bed/multiz10way.v8/cons
    # sort post-prob files by chrom position using filename, then 
    #  use wigEncode to create binary files for wiggle
    find /cluster/bluearc/hg17/multiz10way.v8/phastCons/POSTPROBS \
        -name "*.pp.gz" | sort -t\. -k2,2n | xargs zcat | \
        wigEncode stdin phastCons10way.wig phastCons10way.wib

    hgWiggle -db=hg17 -doHistogram -hBinSize=0.001 \
        -hBinCount=1000 -hMinVal=0.0 -db=hg17 phastCons >histo.8way.data
    hgWiggle -db=hg17 -doHistogram -hBinSize=0.001 \
        -hBinCount=1000 -hMinVal=0.0 -db=hg17 phastCons10way >histo.10way.data
    hgWiggle -db=hg17 -doStats \
         phastCons > stats.8way.data
    hgWiggle -db=hg17 -doStats \
         phastCons10way > stats.10way.data

    # Load gbdb and database with wiggle.
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz10way.v8/cons
    set wibDir = /gbdb/hg17/multiz10way/wib/phastCons10way
    mkdir -p $wibDir
    ln -s `pwd`/phastCons10way.wib $wibDir
    hgLoadWiggle hg17 phastCons10way phastCons10way.wig \
                -pathPrefix=$wibDir

    # create tree image:
        # edit tree.nh to create species.nh with common names
        /cluster/bin/phast/draw_tree -b -s species.nh > species10.ps
        # photoshop to enhance, then save as gif/jpg
        cp /cluster/data/hg17/bed/multiz10way.v8/species10.jpg \
            /usr/local/apache/htdocs/images/phylo/10way.jpg

    # get stats on the track
    ssh hgwdev
        
    featureBits hg17 -enrichment refGene:cds phastConsElements10way
        # refGene:cds 1.020%, phastConsElements10way 5.003%, both 0.711%, cover 69.73%, enrich 13.94x
        # compare to previous elements (generated from 8way)
    featureBits hg17 -enrichment refGene:cds phastConsElements
        # refGene:cds 1.020%, phastConsElements 4.810%, both 0.747%, cover 73.22%, enrich 15.22x

    # see how gluing reduces number of alignments
    ssh eieio
    cd /cluster/data/hg17/bed/multiz10way.v8
    mkdir stats
    grep score maf/chr22.maf | grep -v 0.0 | wc -l
        #179576
    grep score mafGlued/chr22.maf | grep -v 0.0 | wc -l
        #110550

    # look at distribution of alignment sizes after gluing
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz10way.v8
    mkdir mafTemp
    ln -s `pwd`/maf/chr1.maf mafTemp
    # load temp table
    hgLoadMaf hg17 -pathPrefix=mafTemp multiz10wayChr1
        #Loaded 1246727 mafs
    # again, compare to glued:
    echo "SELECT COUNT(*) FROM multiz10way"
        # 738030
        # again, ~40% fewer

    cd stats
    echo "SELECT chromEnd - chromStart FROM multiz10way WHERE chrom='chr1'" | \
        hgsql -N hg17 | sort -n > chr1.maf.glued.sizes
    echo "SELECT chromEnd - chromStart FROM multiz10wayChr1"| \
        hgsql -N hg17 | sort -n > chr1.maf.sizes
    # cleanup
    hgsql hg17 -e "DROP TABLE multiz10wayChr1"
    rm -fr ../mafTemp

    # coverage of multiple alignment, and pairs
    ssh kolossus
    cd /cluster/data/hg17/bed/multiz10way.v8
    cd stats
    nice mafRanges -notAllOGap ../mafGlued/chr1.maf hg17 \
                hg17.chr1.mafRanges.bed
    nice mafRanges -notAllOGap /cluster/data/hg17/bed/multiz8way/maf/chr1.maf \
                hg17 hg17.8way.chr1.mafRanges.bed
    foreach db (panTro1 canFam1 mm5 rn3 galGal2 xenTro1 fr1 tetNig1 danRer1)
        echo $db
        nice mafRanges /cluster/data/hg17/bed/blastz.$db/mafNet/chr1.*maf \
               -notAllOGap hg17 $db.chr1.mafRanges.bed
        ls /cluster/data/hg17/bed/blastz.$db/mafNet/chr1.*maf
    end

    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz10way.v8/stats
    nice featureBits -chrom=chr1 hg17 refGene:cds hg17.chr1.mafRanges.bed -enrichment
# refGene:cds 1.308%, hg17.chr1.mafRanges.bed 95.725%, both 1.307%, cover 99.94%, enrich 1.04x
    nice featureBits -chrom=chr1 hg17 refGene:cds hg17.8way.chr1.mafRanges.bed -enrichment
# refGene:cds 1.308%, hg17.8way.chr1.mafRanges.bed 95.742%, both 1.307%, cover 99.97%, enrich 1.04x

    foreach db (panTro1 canFam1 mm5 rn3 galGal2 xenTro1 fr1 tetNig1 danRer1)
        nice featureBits -chrom=chr1 -enrichment hg17 refGene:cds $db.chr1.mafRanges.bed
    end

#refGene:cds 1.308%, panTro1.chr1.mafRanges.bed 93.472%, both 1.264%, cover 96.65%, enrich 1.03x
#refGene:cds 1.308%, canFam1.chr1.mafRanges.bed 55.377%, both 1.277%, cover 97.64%, enrich 1.76x
#refGene:cds 1.308%, mm5.chr1.mafRanges.bed 37.342%, both 1.280%, cover 97.92%, enrich 2.62x
#refGene:cds 1.308%, rn3.chr1.mafRanges.bed 35.429%, both 1.257%, cover 96.14%, enrich 2.71x
#refGene:cds 1.308%, galGal2.chr1.mafRanges.bed 3.840%, both 0.936%, cover 71.61%, enrich 18.65x
#refGene:cds 1.308%, xenTro1.chr1.mafRanges.bed 3.059%, both 0.881%, cover 67.36%, enrich 22.02x
#refGene:cds 1.308%, fr1.chr1.mafRanges.bed 1.892%, both 0.854%, cover 65.29%, enrich 34.50x
#refGene:cds 1.308%, tetNig1.chr1.mafRanges.bed 1.384%, both 0.805%, cover 61.57%, enrich 44.50x
#refGene:cds 1.308%, danRer1.chr1.mafRanges.bed 2.716%, both 0.847%, cover 64.81%, enrich 23.86x


# MAKE HG17-RN3 OVER.CHAIN FOR LIFTOVER  (DONE 1/25/05 angie)
    ssh kolossus
    set chainDir = /cluster/data/hg17/bed/blastz.rn3/axtChain
    netChainSubset $chainDir/rat.net.gz $chainDir/all.chain.gz \
      /cluster/data/hg17/bed/bedOver/hg17ToRn3.over.chain
    

# MAKE HG17-GALGAL2 OVER.CHAIN FOR LIFTOVER  (DONE 1/25/05 angie)
    ssh kolossus
    set chainDir = /cluster/data/hg17/bed/blastz.galGal2/axtChain
    netChainSubset $chainDir/human.net $chainDir/all.chain \
      /cluster/data/hg17/bed/bedOver/hg17ToGalGal2.over.chain
    

# DOWNLOADS FOR 10-WAY MULTIZ (2005-01-24 kate)
#   Use "glued" mafs

    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17
    mkdir -p multiz10way
    cd multiz10way
    foreach f (/cluster/data/hg17/bed/multiz10way.v8/mafGlued/*.maf)
        set c = $f:r:t
        echo $c
        nice gzip -c $f > $c.maf.gz
    end
    # copy README and edit

    # Create upstream files for download 
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz10way.v8
    echo hg17 panTro1 mm5 rn3 canFam1 galGal2 xenTro1 fr1 tetNig1 danRer1 > org.txt
    # mafFrags takes a while
    foreach i (1000 2000 5000)
        echo "making upstream$i.maf"
        nice featureBits hg17 refGene:upstream:$i -fa=/dev/null -bed=up.bad
        awk -F '\t' '{printf("%s\t%s\t%s\t%s\t%s\t%s\n", $1, $2, $3, substr($4, 0, 9), 0, $5)}' up.bad > up.bed
        rm up.bad
        nice mafFrags hg17 multiz10way up.bed upstream$i.maf -orgs=org.txt
        rm up.bed
    end
    ssh eieio
    cd /cluster/data/hg17/bed/multiz10way.v8
    nice gzip upstream{1000,2000,5000}.maf
        # 6 mins.
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17
    mv /cluster/data/hg17/bed/multiz10way.v8/upstream*.maf.gz multiz10way
    cd multiz10way
    md5sum *.gz > md5sum.txt

    #	Create histogram of this phastCons data  (Hiram - 2005-02-07)
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz.2004-12-22/cons
    time hgWiggle -doHistogram \
        -hBinSize=0.001 -hBinCount=1000 -hMinVal=0.0 -verbose=2 \
	    -db=hg17 phastCons > histogram.data 2>&1
    #	34 minutes

    cat << '_EOF_' > histo.gp
set terminal png small color \
        x000000 xffffff xc000ff x66ff66 xffff00 xff0000 xff0000
set size 1.4, 0.8
set key left box
set grid noxtics
set grid ytics
set title " Hg17 Histogram phastCons track"
set xlabel "Hg17 phastCons score"
set ylabel "p-Value"
set y2label "Cumulative Probability Distribution"
set y2range [0:1]
set y2tics

plot "histogram.data" using 2:5 title " pValue" with impulses, \
        "histogram.data" using 2:7 axes x1y2 title " CPD" with lines
'_EOF_'
    
    gnuplot histo.gp > histo.png
    display histo.png &


# BLASTZ BOREOEUTHERIAN (BOREUT1) (DONE 1/29/05 braney)
    ssh kk
    mkdir /cluster/data/borEut1/bed/zb.hg17 
    ln -s /cluster/data/borEut1/bed/zb.hg17 /cluster/data/hg17/bed/blastz.borEut1
    cd /cluster/data/hg17/bed/blastz.borEut1
    # Use default (Human-Mouse) settings for starters.
    cat << '_EOF_' > DEF
# human vs. dog
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin

ALIGN=blastz-run
BLASTZ=blastz

# Default
BLASTZ_H=2000
BLASTZ_ABRIDGE_REPEATS=1

# TARGET: Human
SEQ1_DIR=/scratch/hg/gs.18/build35/bothMaskedNibs
SEQ1_RMSK=
SEQ1_FLAG=
SEQ1_SMSK=/scratch/hg/gs.18/build35/linSpecRep.notInDog
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Dog
SEQ2_DIR=/iscratch/i/borEut1/nib
SEQ2_RMSK=
SEQ2_FLAG=
SEQ2_SMSK=/scratch/hg/canFam1/linSpecRep.notInHuman
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/blastz.borEut1

DEF=$BASE/DEF
RAW=$BASE/raw
CDBDIR=$BASE
SEQ1_LEN=$BASE/S1.len
SEQ2_LEN=$BASE/S2.len
'_EOF_'
    # << this line keeps emacs coloring happy

    /cluster/data/hg17/jkStuff/BlastZ_run0.sh
    cd run.0
    para push
# Completed: 2728 of 2728 jobs
# CPU time in finished jobs:     621440s   10357.34m   172.62h    7.19d  0.020 y
# IO & Wait Time:                 19079s     317.98m     5.30h    0.22d  0.001 y
# Average job time:                 235s       3.91m     0.07h    0.00d
# Longest job:                     2340s      39.00m     0.65h    0.03d
# Submission to last job:          2837s      47.28m     0.79h    0.03d

    ssh kki
    cd /cluster/data/hg17/bed/blastz.borEut1
    /cluster/data/hg17/jkStuff/BlastZ_run1.sh

    cd run.1
    para push
# Completed: 341 of 341 jobs
# CPU time in finished jobs:         95s       1.58m     0.03h    0.00d  0.000 y
# IO & Wait Time:                   825s      13.75m     0.23h    0.01d  0.000 y
# Average job time:                   3s       0.04m     0.00h    0.00d
# Longest job:                       10s       0.17m     0.00h    0.00d
# Submission to last job:            73s       1.22m     0.02h    0.00d

    ssh kk
    cd /cluster/data/hg17/bed/blastz.borEut1
    /cluster/data/hg17/jkStuff/BlastZ_run2.sh
    cd run.2
    para push

# /cluster/data/hg17/bed/blastz.borEut1/axtChrom/chr18_random.axt is empty
# /cluster/data/hg17/bed/blastz.borEut1/axtChrom/chr19_random.axt is empty
# ..
# Completed: 44 of 46 jobs
# Crashed: 2 jobs
# CPU time in finished jobs:        104s       1.73m     0.03h    0.00d  0.000 y
# IO & Wait Time:                   482s       8.04m     0.13h    0.01d  0.000 y
# Average job time:                  13s       0.22m     0.00h    0.00d
# Longest job:                      134s       2.23m     0.04h    0.00d
# Submission to last job:           142s       2.37m     0.04h    0.00d

# END BLASTZ BOREOEUTHERIAN


##########################################################################
# MAKE tfbsConsSites and tfbsConsFactors for TFBS conserved track (DONE braney 1/15/05)
# Questions?  weirauch@soe.ucsc.edu or braney@soe.ucsc.edu
#	tfbsConsSites table reloaded 2006-11-03 - Hiram - see below:
## reload tfbsCons table - it was based on a newer version of tfbs names that

ssh hgwdev
mkdir /cluster/data/hg17/bed/tfbsCons
cd /cluster/data/hg17/bed/tfbsCons

# Define all parameters in 'PARAMS.txt'
# Define all chromosomes in 'CHROMS.txt'
# Get tfbsConsUtils.tar.gz from Matt Weirauch with Perl scripts  weirauch@soe.ucsc.edu
set tarfile=/cluster/data/hg17/bed/tfbsCons/tfbsConsUtils.tar.gz
tar zxf $tarfile

nice ./getRefseqStats.pl &
nice ./getBatchQueries.pl &

ssh kk
mkdir /cluster/bluearc/braney/tfloc
# Copy ./tmp/ctfbs_batch_list.txt to this dir
# Copy ./scripts/doit to this dir
para create ctfbs_batch_list.txt
para try
para push

# When the run is done (within a day or so), the results will be in individual dirs, one for each chromosome.

ssh kksilo (or hgwdev, or whatever)
nice ./getBedFile.pl &

hgLoadBed -noSort hg17 tfbsConsSites \
	-sqlTable=$HOME/kent/src/hg/lib/tfbsConsSites.sql \
	tfbsConsSites.bed -tab
hgLoadBed -noSort hg17 tfbsConsFactors \
	-sqlTable=$HOME/kent/src/hg/lib/tfbsConsFactors.sql \
	tfbsConsFactors.bed -tab

# Feel free to delete or gzip anything in ./tmp
#  (particularly the huge .maf and .bed files)
#   after the final two bed files are sucessfully loaded

##########################################################################
# CHICKEN RECIPROCAL-BEST NET FOR STRINGENT LIFTOVER (DONE 2/3/05 angie)
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.galGal2/axtChain
    # Run chainNet again, this time keeping both of its outputs:
    chainPreNet all.chain ../S1.len ../S2.len stdout \
    | chainNet stdin ../S1.len ../S2.len h_g.net g_h.net
    # Get the chicken chains from the chicken-referenced (but human-centric)
    # net:
    chainSwap all.chain g_h.chain
    netChainSubset g_h.net g_h.chain stdout \
    | chainSort stdin g_h.subset.chain
    # Net those (sorted) chicken chains, and keep both outputs, to get 
    # reciprocal best nets referenced to both species:
    chainPreNet g_h.subset.chain ../S2.len ../S1.len stdout \
    | chainNet stdin ../S2.len ../S1.len g_h.rbest.net h_g.rbest.net
    # Get the chains from the recip-best nets for stringent liftOver:
    netChainSubset g_h.rbest.net g_h.chain galGal2ToHg17.rbest.over.chain
    netChainSubset h_g.rbest.net all.chain hg17ToGalGal2.rbest.over.chain



####### RE-BUILD RGD HUMAN QTL TRACKS (DONE 2/5/05 Fan) ##############

mkdir -p /cluster/store8/rgd/human050205
rm /cluster/data/hg17/bed/rgdQtl
ln -s /cluster/store8/rgd/human050205 /cluster/data/hg17/bed/rgdQtl
cd /cluster/data/hg17/bed/rgdQtl

# download data files from RGD

wget --timestamp ftp://rgd.mcw.edu/pub/RGD_genome_annotations/human/human_QTL.gff

# remove extra line feed character at the end of lines

# !!! manually corrected the line of AASTH7_H because chromStart is greater than chrEnd

rmLf human_QTL.gff > rgdQtl.gff

# create rgdQtl.tab
awk '{print $1"\t"$4"\t"$5"\t"$10}'  rgdQtl.gff |sed -e 's/Chr/chr/g'| \
sed -e 's/"//g' |sed -e 's/RGD://g' | sed -e 's/;//g' > rgdQtl.tab

# create rgdQtlLink.tab

awk '{printf "%s\t%s\t", $12, $10; for (i = 14;i <= NF; ++i ) {printf "%s ", $i} printf "\n"} ' rgdQtl.gff | \
sed -e 's/"//g'| sed -e 's/RGD://g' | sed -e 's/;//g'| sed -e 's/Note//g' > rgdQtlLink.tab

# load rgdQtl table
hgLoadBed hg17 rgdQtl rgdQtl.tab

# check rgdQtl table
checkTableCoords hg17 rgdQtl

# load rgdQtlLink table
hgsql hg17 -e "drop table hg17.rgdQtlLink;"
hgsql hg17 <~/kent/src/hg/lib/rgdQtlLink.sql
hgsql hg17 -e 'load data local infile "rgdQtlLink.tab" into table hg17.rgdQtlLink;'

# updated trackDb.ra under /kent/src/hg/makeDb/trackDb/human/hg17 and
# added rgdQtl.html.

# GENOSCOPE TETRAODON (tetNig1) ECORES (DONE, 2005-02-08, hartera)
    ssh eieio
    mkdir -p /cluster/data/hg17/bed/ecoresTetNig1
    cd /cluster/data/hg17/bed/ecoresTetNig1
                                                                                
    wget --timestamp \
         http://www.genoscope.cns.fr/externe//4ucsc/ExofishHs35Tnig1
    # this is in gff format
    # remove "Ecotig" from name field
    sed -e 's/Ecotig EG/EG/g' ExofishHs35Tnig1 > ExofishHs35Tnig1.gff
    # need to have tabs between fields not a space to load file into table
    sed -e 's/ /\t/g' ExofishHs35Tnig1.gff > Hs35Tnig1format.gff
    # if "ecore" is changed to "CDS" and "ecotig" to "transcript" this loads
    # correctly into the table.
    sed -e 's/ecore/CDS/' Hs35Tnig1format.gff | sed -e 's/ecotig/transcript/' \
           > Hg17vstetNig1.gff
    # add "chr" in front of the chromsome name in first field (2005-02-08)
    perl -pi.bak -e 's/^([0-9XYM]{1,2})/chr$1/' Hg17vstetNig1.gff
    rm *.bak
    # need to reload table

    ssh hgwdev
    cd /cluster/data/hg17/bed/ecoresTetNig1
    echo 'drop table ecoresTetNig1;' | hgsql hg17
    nice ldHgGene hg17 ecoresTetNig1 Hg17vstetNig1.gff
    # Read 40172 transcripts in 186032 lines in 1 files
    #  40172 groups 42 seqs 1 sources 2 feature types
    # 40172 gene predictions
# added ecoresTetNig1 entry to trackDb.ra in trackDb/human
# and created ecoresTetNig1.html.  Genoscope will not be maintaining this
# newest data in their Exofish comparative browser display.

# UPDATE kgSpAlias TABLE WITH NEW UNIPROT DISPLAY ID ENTRIES (done 2/11/05 Fan)
# Add new human protein display IDs to the alias table to support user search
    
    ssh hgwdev
    cd /cluster/data/hg17/bed/pb
    mkdir newDisplayId
    cd newDisplayId
 
    hgsql proteome -e 'select hg17.kgSpAlias.kgID, hg17.kgSpAlias.SpID, spOldNew.newDisplayId from spOldNew, hg17.kgSpAlias where spOldNew.acc=hg17.kgSpAlias.spID and oldDisplayId != newDisplayId' |sort -u >hg17.tab
    
    hgsql hg17 -e 'load data local infile "hg17.tab" into table hg17.kgSpAlias'

# UPDATE kgProtAlias TABLE WITH NEW UNIPROT DISPLAY ID ENTRIES (done 2/11/05 Fan)
# Add new hg17 protein display IDs to the alias table to support user search
    
    ssh hgwdev
    cd /cluster/data/hg17/bed/pb/newDisplayId

     hgsql proteome -e 'select hg17.kgSpAlias.kgID,spOldNew.oldDisplayId,spOldNew.newDisplayId from spOldNew, hg17.kgSpAlias where spOldNew.acc=hg17.kgSpAlias.spID and oldDisplayId != newDisplayId' |sort -u >hg17.kgProtAlias.tab

# get rid of the header line at the end of the file
    vi hg17.kgProtAlias.tab 

    hgsql hg17 -e 'load data local infile "hg17.kgProtAlias.tab" into table hg17.kgProtAlias'



# BLASTZ HUMAN TARGET, COW QUERY (DONE, Nov. 2004 - Jan. 2005, Heather)
  ssh kk
  # use /cluster/data/bosTau1 because more disk space there
  cd /cluster/data/bosTau1/bed
  mkdir zb.hg17

  # create DEF file 
  # for now, not doing ABRIDGE_REPEATS
  # this means I don't need to create lineage specific repeats
  # This is because blastz-run wouldn't take advantage of these
  # because my query is in scaffolds

# human vs. cow
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/cluster/home/angie/schwartzbin

ALIGN=blastz-run
BLASTZ=blastz

# Default
BLASTZ_H=2000
BLASTZ_ABRIDGE_REPEATS=0

# TARGET: Human
SEQ1_DIR=/scratch/hg/gs.18/build35/bothMaskedNibs
#SEQ1_DIR=/iscratch/i/hg17/bothMaskedNibs
SEQ1_RMSK=
SEQ1_FLAG=
SEQ1_SMSK=
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Cow
SEQ2_DIR=/iscratch/i/bosTau1/splitDir
SEQ2_RMSK=
SEQ2_FLAG=
SEQ2_SMSK=
SEQ2_IN_CONTIGS=1
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/zb.hg17

DEF=$BASE/DEF
RAW=$BASE/raw
CDBDIR=$BASE
SEQ1_LEN=$BASE/S1.len
SEQ2_LEN=$BASE/S2.len

'_EOF_'
  # << this line keeps emacs coloring happy
  bash
  cd /cluster/data/bosTau1/bed/zb.hg17
  source DEF
  mkdir $RAW run.0
  # create S2.len so make-joblist doesn't have to
  /cluster/bin/scripts/blastz-make-joblist $DEF > $BASE/run.0/j
  sh ./xdir.sh
  cd run.0
  # check how many lines in j
  sed -e 's@^blastz-run@/cluster/bin/penn/blastz-run@' j > jobList
  para create jobList
  para try, para check, para push, para check....

  # convert out to lav
  ssh kki
  cd /cluster/data/bosTau1/bed/zb.hg17
  # run bash shell if not running it already
  source DEF
  mkdir -p $BASE/run.1
  mkdir -p $BASE/lav
  # create a new job list to convert out files to lav
  /cluster/bin/scripts/blastz-make-out2lav $DEF $BASE > run.1/jobList
  cd run.1
  # make sure the job list is OK
  wc -l jobList
  head jobList
  para create jobList
  para try
  para check
  para push

  # lavToAxt
  ssh kksilo
  cd /cluster/data/bosTau1/bed/zb.hg17
  mkdir axtTemp
  cd lav
  foreach i (*)
      catDir $i | lavToAxt stdin /cluster/data/hg17/nib \
        /cluster/data/bosTau1/bosTau1.2bit ../axtTemp/$i.axt
      echo done $i
  end

  # axtChain
  ssh kki
  cd /cluster/data/bosTau1/bed/zb.hg17
  mkdir -p axtChain/run1
  cd axtChain/run1
  mkdir out chainRaw
  ls -1S /cluster/data/bosTau1/bed/zb.hg17/axtTemp/*.axt > input.lst

  cat << '_EOF_' > gsub
  #LOOP
  doChain {check in exists $(path1)} {check out line+ chainRaw/$(root1).chain} {check out exists out/$(root1).out}
  #ENDLOOP
  '_EOF_'
  # << this line makes emacs coloring happy
    
  cat << '_EOF_' > doChain
  #!/bin/csh
'_EOF_'
  axtChain $1 /iscratch/i/hg17/bothMaskedNibs /iscratch/i/bosTau1/nib/bosTau1.2bit $2 > $3
    
  # << this line makes emacs coloring happy
  chmod a+x doChain
  gensub2 input.lst single gsub jobList
  para create jobList
  para try
  para check
  para push

  # Completed: 46 of 46 jobs
  # Average job time:                  83s       1.39m     0.02h    0.00d
  # Longest job:                     1240s      20.67m     0.34h    0.01d
  # Submission to last job:          1326s      22.10m     0.37h    0.02d

  # mergesort
  ssh kksilo
  cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
  chainMergeSort run1/chainRaw/*.chain > all.chain.jan3

  # chainAntiRepeat
  ssh kksilo
  cd /cluster/data/bosTau1/bed/zb.hg17/axtChain/run1
  mkdir chainAntiRepeat
  # test with just one
  chainAntiRepeat /cluster/store5/gs.18/build35/nib /cluster/data/bosTau1/bosTau1.2bit \
    chainRaw/chr18.chain chainAntiRepeat/chr18.chain
  # do them all
  foreach f (chainRaw/*.chain)
    set f1 = $f:t
    echo $f1
      chainAntiRepeat /cluster/store5/gs.18/build35/nib /cluster/data/bosTau1/bosTau1.2bit \
        $f chainAntiRepeat/$f1
  end

  # mergesort again
  ssh kksilo
  cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
  chainMergeSort run1/chainAntiRepeat/*.chain > all.chain.jan5
  gzip all.chain.jan3

  # split
  mkdir chain
  chainSplit chain all.chain.jan5

  # look at the distribution
  foreach f (chain/*.chain)
    grep chain $f | awk '{print $2;}' | sort -nr > /tmp/score.$f:t:r
    echo $f:t:r
    textHistogram -binSize=5000 /tmp/score.$f:t:r
    echo ""
  end
  # see files histogram.out and histogram.interesting

  # run chainFilter
  chainFilter -minScore=5000 all.chain.jan5 > all.chain.jan5.filtered
  gzip all.chain.jan5

  # split
  rm chain/*
  chainSplit chain all.chain.jan5.filtered
  gzip all.chain.jan5.filtered

  # load
  ssh hgwdev
  cd /cluster/data/bosTau1/bed/zb.hg17/axtChain/chain
  foreach i (*.chain)
    set c = $i:r
    echo loading $c
    hgLoadChain hg17 ${c}_chainBosTau1 $i
  end

    # featureBits -chrom=chr1 hg17 chainBosTau1Link
    # 103272818 bases of 222827847 (46.346%) in intersection
    # featureBits -chrom=chr2 hg17 chainBosTau1Link
    # 105920345 bases of 237506229 (44.597%) in intersection
    # featureBits -chrom=chr3 hg17 chainBosTau1Link
    # 89582887 bases of 194635740 (46.026%) in intersection
    # featureBits -chrom=chr4 hg17 chainBosTau1Link
    # 77513949 bases of 187161218 (41.416%) in intersection
    # featureBits -chrom=chr5 hg17 chainBosTau1Link
    # 80428726 bases of 177702766 (45.260%) in intersection
    # featureBits -chrom=chr6 hg17 chainBosTau1Link
    # 71830264 bases of 167317699 (42.930%) in intersection
    # featureBits -chrom=chr7 hg17 chainBosTau1Link
    # 64561289 bases of 154759139 (41.717%) in intersection
    # featureBits -chrom=chr8 hg17 chainBosTau1Link
    # 55896735 bases of 142612826 (39.195%) in intersection
    # featureBits -chrom=chr9 hg17 chainBosTau1Link
    # 52068957 bases of 117781268 (44.208%) in intersection
    # featureBits -chrom=chr10 hg17 chainBosTau1Link
    # 57427282 bases of 131613628 (43.633%) in intersection
    # featureBits -chrom=chr11 hg17 chainBosTau1Link
    # 58412709 bases of 131130853 (44.545%) in intersection
    # featureBits -chrom=chr12 hg17 chainBosTau1Link
    # 56076163 bases of 130259811 (43.049%) in intersection
    # featureBits -chrom=chr13 hg17 chainBosTau1Link
    # 37951944 bases of 95559980 (39.715%) in intersection
    # featureBits -chrom=chr14 hg17 chainBosTau1Link
    # 39896970 bases of 88290585 (45.188%) in intersection
    # featureBits -chrom=chr15 hg17 chainBosTau1Link
    # 37507979 bases of 81341915 (46.112%) in intersection
    # featureBits -chrom=chr16 hg17 chainBosTau1Link
    # 33883573 bases of 78884754 (42.953%) in intersection
    # featureBits -chrom=chr17 hg17 chainBosTau1Link
    # 31871034 bases of 77800220 (40.965%) in intersection
    # featureBits -chrom=chr18 hg17 chainBosTau1Link
    # 30359555 bases of 74656155 (40.666%) in intersection



# NET 
# run in stages to avoid memory problems
  ssh kolossus
  cd /cluster/data/bosTau1/bed/zb.hg17/axtChain

  # PRE
  /cluster/bin/x86_64/chainPreNet all.chain.jan5.filtered ../S1.len ../S2.len chainPreNet.out

  # chainNet
  /cluster/bin/x86_64/chainNet chainPreNet.out \
    -minSpace=1 ../S1.len ../S2.len bosTau1.net.raw /dev/null

  # syntenic (using revision 1.6)
  /cluster/home/heather/bin/x86_64/netSyntenic bosTau1.net.raw bosTau1.net.syn
  # memory usage 2757492736, utime 13404 s/100, stime 616

  # backup/compress
  ssh kksilo
  cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
  gzip bosTau1.net.raw
  cp bosTau1.net.syn bosTau1.net.syn.backup

  # netClass
  # takes about 4 hours
  ssh hgwdev
  cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
  netClass -noAr bosTau1.net.syn hg17 bosTau1 bosTau1.net

  # backups
  ssh kksilo
  cp bosTau1.net bosTau1.net.backup
  rm bosTau1.net.syn.backup

  # load
  ssh hgwdev
  cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
  netFilter -minGap=10 bosTau1.net | hgLoadNet hg17 netBosTau1 stdin
  rm bosTau1.net.backup

  # index has NULL cardinality; analyze to fix
  hgsql hg17
  analyze table netBosTau1

  # generate axts 
  ssh kksilo
  cd /cluster/data/bosTau1/bed/zb.hg17
  mkdir axtNet

  # split first (not required?)
  cd axtChain
  mkdir net
  netSplit bosTau1.net.syn net
  cd net
  foreach i (*.net)
    netToAxt $i ../chain/$i:r.chain /cluster/data/hg17/nib /cluster/data/bosTau1/bosTau1.2bit ../../axtNet/$i:r.axt
  end
  gzip bosTau1.net.syn
  gzip bosTau1.net

  # axtSort (takes about 5 minutes)
  ssh kksilo
  cd /cluster/data/bosTau1/bed/zb.hg17
  mkdir axtNetSort
  foreach f ( axtNet/*.axt )
    set c = $f:t:r
    echo "axtSort on $c"
    axtSort $f axtNetSort/$c.axt
  end

  # make maf files
  mkdir mafNet
  foreach f (axtNetSort/*.axt)
    set c = $f:t:r
    echo "axtToMaf on $c"
    axtToMaf $f /cluster/data/hg17/chrom.sizes /cluster/data/bosTau1/chrom.sizes mafNet/$c.maf -tPrefix=hg17. -qPrefix=bosTau1.
  end

# MAKE VSBOSTAU1 DOWNLOADABLES (DONE Feb. 15, 2005 Heather)
    ssh hgwdev
    cd /usr/local/apache/htdocs/goldenPath/hg17
    mkdir vsBosTau1
    cd vsBosTau1
    mkdir axtNet
    cd /cluster/data/bosTau1/bed/zb.hg17/axtChain
    cp -p all.chain.gz  /usr/local/apache/htdocs/goldenPath/hg17/vsBosTau1/cow.chain.gz
    cp -p bosTau1.net.gz /usr/local/apache/htdocs/goldenPath/hg17/vsBosTau1/cow.net.gz
    cd ../axtNet
    cp -p * /usr/local/apache/htdocs/goldenPath/hg17/vsBosTau1/axtNet

    cd /usr/local/apache/htdocs/goldenPath/hg17/vsBosTau1
    # Make a README.txt which explains the files & formats.
    md5sum *.gz > md5sum.txt
    cd axtNet
    md5sum *.gz > md5sum.txt

# YALE PSEUDOGENES (started Robert Baertsch, finished JK 2/21/05)
    ssh hgwdev
    cd /cluster/data/hg17/bed
    mkdir pseudoYale
    cd pseudoYale
    # Place file obtained from Mark Gerstein at yale in pseudoYale.gtf
    ldHgGene hg17 pseudoYale pseudoYale.gtf
    # Note - I'm guessing how this goes.  Robert left no record. -jk

# added xenoRefGene track (markd ~2005-02-20)
    add to /cluster/data/genbank/genbank.con
    hg17.refseq.mrna.xeno.load  = yes
    hg17.refseq.mrna.xeno.loadDesc = yes

# BUILD ccdsGene and ccdsInfo tables (markd 2005-02-25)
    # download files to the genbank data area, as this will eventually
    # be done automatically as part of the genbank build process.
    cd  /cluster/data/genbank
    mkdir -p data/ccds/hg17/2005-02-25
    cd data/ccds/hg17/2005-02-25

    # get the basic text dumps of the data, rather than the database dumps
    wget -r ftp://ftp.ncbi.nlm.nih.gov/pub/CCDS/
    # ends up with:
        About-NcbiHinxton.txt
        NcbiHinxton.txt
        NcbiHinxtonAllAccessions.txt

    # this is a preliminary release, it contained 2 PAR genes that had
    # bad coordinates and 7 genes that were determined at be pseudogenes
    # at the last minute.  The accessions for these 9 genes were
    # placed in skip.ccds and then removed:
    fgrep -v -f skip.ccds  /scratch/markd/gene-sets/ncbiDb/set1.5/NcbiHinxtonAllAccessions.txt >  /scratch/markd/gene-sets/ncbiDb/set1.5/NcbiHinxtonAllAccessions.cleaned.txt

    # create the tab files to load in the database
    /cluster/data/genbank/bin/i386/ccdsImport NcbiHinxtonAllAccessions.cleaned.txt ccdsGene.gp ccdsInfo.tab

    # load ccdsInfo
    hgsql hg17 <../../../../../lib/ccdsInfo.sql 
    hgsql -e 'load data local infile "ccdsInfo.tab" into table ccdsInfo'  hg17

    # load cdsGene.gp and check
    ldHgGene -predTab -genePredExt hg17 ccdsGene ccdsGene.gp 
    checkTableCoords hg17 -verbose=2 ccdsGene
    rm *.tab
    gzip -9 NcbiHinxton*.txt
  
    
# BUILD refSeqKg TABLE TO SUPPORT CCDS GENES (RE-DONE, Fan 2/26/05)

    hgsql hg17 -N -e "select * from knownGene" >kg.gp
    hgsql hg17 -N -e "select * from refGene"   >ref.gp

    overlapSelect -inCds -strand  -idOutput -fraction=fraction.out -selectCds -overlapSimilarity=0.90 -selectFmt=genePred -inFmt=genePred kg.gp ref.gp refSeqKg.90.tab
    cat fraction.out|sort -u >refSeqKg.tab

    hgsql hg17 -e 'drop table refSeqKg'
    hgsql hg17 < ~/src/hg/lib/refSeqKg.sql
    hgsql hg17 -e 'load data local infile "refSeqKg.tab" into table refSeqKg'

   rm fraction.out

# BUILD ccdsGene and ccdsInfo tables  (markd, reone 2005-03-17)
    cd /cluster/store5/genbank/data/ccds/hg17
    wget ftp://ftp.ncbi.nlm.nih.gov/pub/hcds/Hs35.1/CDSTrackDB/CCDS.20050303.tar.gz
    mkdir /scratch/tmp/ccds
    cd /scratch/tmp/ccds
    tar -zxf  /cluster/store5/genbank/data/ccds/hg17/CCDS.20050303.tar.gz     

    # import ccds database tables
    hgsql -e 'create database ccds'
    hgsql ccds </cluster/data/genbank/etc/createTables.sql 
    hgsql ccds </cluster/data/genbank/etc/createKeys.sql 
    /cluster/data/genbank/bin/i386/ccdsImport ccds data/[A-Z]*.txt

    # create and load ccdsGene and ccdsInfo tables from imported database
    /cluster/data/genbank/bin/i386/ccdsMkTables -loadDb ccds hg17 ccdsInfo ccdsGene

    # refSeqKg table

    hgsql -N -e "select * from knownGene" hg17 >kg.gp
    hgsql -N -e "select * from refGene" hg17 >ref.gp

    overlapSelect -statsOutput -strand -inCds -selectCds -overlapSimilarity=0.90 kg.gp ref.gp stdout | tail +2 | sort -u >refSeqKg.tab

    hgsql hg17 -e 'drop table refSeqKg'
    hgsql hg17 < ~/compbio/kent/src/hg/lib/refSeqKg.sql
    hgsql hg17 -e 'load data local infile "refSeqKg.tab" into table refSeqKg'
    cd ..
    rm -r ccds


# COW BACENDS (Done, Heather, Mar. 21, 2005)
 
ssh hgwdev
cd /cluster/data/hg17/bed
mkdir bacendsCow
cd bacendsCow
 
# Obtain GFF file from Denis; unzip into BACendhg15.gff

# Convert into BED 6:
 
  makebed.pl < BACendhg17.gff > BACendhg17.bed
  hgLoadBed -noBin hg17 bacendsCow BACendhg17.bed
  # 53403 warnings
 
# add to kent/src/hg/makeDb/trackDb/human/hg17/trackDb.ra

# make map between ccds and known genes (markd 2005/03/08)
# this should be run whenever either known genes or ccds is updated
/cluster/data/genbank/bin/i386/mkCcdsGeneMap  -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap

# UPDATE WGRNA TRACK (DONE, 2004-12-13, Fan)

# Received updated data file, wg_track_april2005.txt, from Michel Weber by email.

   cut -f 2-10  wg_track_april2005.txt |tail +2 >wg_track_april2005.tab

# Use editor to remove the last blank line.

   hgLoadBed -sqlTable=/cluster/home/fanhsu/hg/lib/wgRna.sql hg17 wgRna wg_track_april2005.tab

# Asked Donna to update Reference section according to Michel's email.  

## refresh vega tracks with vega build30        (done 5/4/04 Robert)
##download vega mysql tables
cd /cluster/store8/ensembl
mkdir vega30_35c
cd vega30_35c
ln /cluster/store8/ensembl/vega30_35c /cluster/data/hg17/bed/vega30 -s

for i in `cat tables` ; do wget -N ftp://ftp.ensembl.org/pub/human-30.35c/data/mysql/homo_sapiens_vega_30_35c/$i.gz ; done
wget -N ftp://ftp.ensembl.org/pub/human-30.35c/data/mysql/homo_sapiens_vega_30_35c/homo_sapiens_vega_30_35c_mysql40_compatible..sql.gz 
gunzip *.gz
##create mysql database
mysql 
create database vega30
use vega30
source homo_sapiens_vega_30_35c_mysql40_compatible.sql
source dropMt.sql
source load.sql
exit


hgsql vega30 -N -B < vegaGene.sql > vegaGene.tab
awk -f vegaGene.awk < vegaGene.tab > vegaGene.gp     
ldHgGene hg17 vegaGene -predTab vegaGene.gp -gtf -genePredExt 
hgsql vega30 -N -B < vegaPseudo.sql > vegaPseudo.tab
awk -f vegaPseudo.awk < vegaPseudo.tab > vegaPseudo.gp     
ldHgGene hg17 vegaPseudoGene -predTab vegaPseudo.gp -gtf -genePredExt 

#load processed pseudogenes
grep Processed vegaPseudo.tab > vegaProcPseudo.tab
awk -f vegaPseudo.awk < vegaProcPseudo.tab > vegaProcPseudo.gp
ldHgGene hg17 vegaProcessedPseudo -predTab vegaProcPseudo.gp -gtf -genePredExt

#load vegaInfo
hgsql vega30 -N -B < vegaGeneInfo.sql > vegaInfo.tab
hgsql vega30 -N -B < vegaPseudoInfo.sql >> vegaInfo.tab

hgsql hg17 -N -B < /cluster/home/baertsch/kent/src/hg/lib/vegaInfo.sql
echo 'load data local infile "vegaInfo.tab" into table vegaInfo' | hgsql hg17 -N -B

#load down to hg16
liftOver vegaGene.gp /gbdb/hg17/liftOver/hg17ToHg16.over.chain vegaGeneHg16.gp unMapped.gp -genePred
liftOver vegaPseudo.gp /gbdb/hg17/liftOver/hg17ToHg16.over.chain vegaPseudoGeneHg16.gp unMappedPseudo.gp -genePred

ldHgGene hg16 vegaGene -predTab vegaGeneHg16.gp -gtf 
ldHgGene hg16 vegaPseudoGene -predTab vegaPseudoGeneHg16.gp -gtf 
echo 'truncate table vegaInfo' | hgsql hg16 -N -B
echo 'load data local infile "vegaInfo.tab" into table vegaInfo' | hgsql hg16 -N -B


#########################################################################
# MOUSE NET/CHAINS MM6 - Info contained in makeMm6.doc (200503 Hiram)


####################################################################################
# RE-BUILD KNOWN GENES TABLES, 2ND TRIAL WITH VARIANT PROTEINS (Started 5/13/05 Fan)

# First build protein databases, sp050415 and proteins050415
# See makeProteins050415.doc for details.

# Create working subdirectories and temporary databases (kgHg17F)

  ssh hgwdev
  cd /cluster/store10/kg
  mkdir kgHg17F  
  ln -s /cluster/store10/kg/kgHg17F /cluster/store6/kgDB/bed/kgHg17F
  ln -s /cluster/store10/kg/kgHg17F /cluster/data/hg17/bed/kgHg17F

  hgsql hg17 -e "create database kgHg17F"   
  hgsql hg17 -e "create database kgHg17FTemp"

  mkdir /cluster/bluearc/kgDB/kgHg17F
  mkdir /cluster/bluearc/kgDB/kgHg17F/protBlat
  ln -s /cluster/bluearc/kgDB/kgHg17F/protBlat /cluster/store10/kg/kgHg17F/protBlat
  cd /cluster/store10/kg/kgHg17F/protBlat

################################################################# 
# VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV
# The protBlat.psl was built during the first KG II build trial
# The results are still valid, except that kgHg17E was used
# instead of kgHg17F

# Create working subdirectories and temporary databases (kgHg17E)

  ssh hgwdev
  cd /cluster/store10/kg
  mkdir kgHg17E  
  ln -s /cluster/store10/kg/kgHg17E /cluster/store6/kgDB/bed/kgHg17E
  ln -s /cluster/store10/kg/kgHg17E /cluster/data/hg17/bed/kgHg17E

  hgsql hg17 -e "create database kgHg17E"   
  hgsql hg17 -e "create database kgHg17ETemp"

  mkdir /cluster/bluearc/kgDB/kgHg17E
  mkdir /cluster/bluearc/kgDB/kgHg17E/protBlat
  ln -s /cluster/bluearc/kgDB/kgHg17E/protBlat /cluster/store10/kg/kgHg17E/protBlat
  cd /cluster/store10/kg/kgHg17E/protBlat

# Get all human protein sequences

  hgsql -N sp050415 -e \
  'select proteins050415.spXref3.accession,protein.val from proteins050415.spXref3,protein where division="9606" and acc=accession' \
  |awk '{print ">" $1;print $2}' >humanProt.fa

# Prepare and perform cluster run for protein/genome alignment

  ssh kk
  cd /cluster/data/hg17/bed/kgHg17E/protBlat
  mkdir prot
  faSplit sequence humanProt.fa 1000 prot/prot
  ls /cluster/bluearc/kgDB/kgHg17E/protBlat/prot/* > prot.lis

  ssh hgwdev
  cd /cluster/data/hg17/bed/kgHg17E/protBlat
  hgsql hg17 -N -e 'select chrom from chromInfo' > chrom.lis
  exit
  
  cat << '_EOF_' > gsub
#LOOP
/cluster/bin/i386/blat -noHead -t=dnax -q=prot /cluster/data/hg17/nib/$(path1).nib $(path2) {check out line+ /cluster/bluearc/kgDB/kgHg17E/protBlat/result/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'

  mkdir result
  gensub2 chrom.lis prot.lis gsub jobList

  para create jobList
  para try
  para check
  para push
  para check ...
# many output .psl will be empty, the warnings are OK.
[kk:protBlat> para check
45494 jobs in batch
0 jobs (including everybody's) in Parasol queue.
Checking finished jobs
tracking errors: 1
crashed: 12643
ranOk: 32850
total jobs in batch: 45494
[kk:protBlat> para time
45494 jobs in batch
0 jobs (including everybody's) in Parasol queue.
Checking finished jobs
Completed: 32850 of 45494 jobs
Crashed: 12643 jobs
para.results: file not found.  paraHub can't write to this dir?
CPU time in finished jobs:   36153510s  602558.50m 10042.64h  418.44d  1.146 y
IO & Wait Time:               1585456s   26424.27m   440.40h   18.35d  0.050 y
Average job time:                1149s      19.15m     0.32h    0.01d
Longest running job:                0s       0.00m     0.00h    0.00d
Longest finished job:          155120s    2585.33m    43.09h    1.80d
Submission to last job:        276342s    4605.70m    76.76h    3.20d

# This cluster run took about 3 days.  Crashed jobs are due to empty BLAT result.  It is OK.

# collect BLAT results

   ssh hgwdev
   cd /cluster/data/hg17/bed/kgHg17E/protBlat

   mkdir result2
   mkdir result3

   cat chrom.lis |sed -e 's/chr/do1 chr/g' >doall

   cat << '_EOF_' > do1.1
echo processing $1
cat result/$1_prot*.psl >result2/$1.psl
'_EOF_'

   cat << '_EOF_' > do1.2
echo processing $1
pslReps -nohead -minCover=0.80 -minAli=0.80 -nearTop=0.002 result2/$1.psl result3/$1.psl /dev/null >>j.out
'_EOF_'

   chmod +x do*

   cp do1.1 do1
   doall
   cp do1.2 do1
   doall

   cat result3/*.psl >protBlat.psl

#^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# The end of protBlat.psl build, using kgHg17E
################################################################################




############################################################################
# This part process the variant splice proteins.

# First build variant splice protein tables.

# Get all variant isoform human protein sequences

  ssh hgwdev
  cd /cluster/data/swissprot/050415/build
  wget --timestamp \
 ftp://us.expasy.org/databases/uniprot/current_release/knowledgebase/complete/uniprot_sprot_varsplic.fasta.gz
  wget --timestamp \
ftp://us.expasy.org/databases/uniprot/current_release/knowledgebase/complete/uniprot_trembl_varsplic.fasta.gz

  gzip -d *varsplic.fasta.gz

  faToTab -type=protein uniprot_trembl_varsplic.fasta splicTrembl.tab
  faToTab -type=protein uniprot_sprot_varsplic.fasta splicSprot.tab

  cat splicTrembl.tab splicSprot.tab >varProtein.tab
  hgsql sp050415 < ~/src/hg/lib/varProtein.sql
  hgsql sp050415 -e 'load data local infile "varProtein.tab" into table varProtein'

  cat varProtein.tab |cut -f 1>j1
  cut -f 1 j1|sed -e 's/-/\t/g' >j2
  paste j1 j2 >splicProt.tab

  hgsql kgHg17FTemp -e 'drop table splicProt'
  hgsql kgHg17FTemp <~/src/hg/lib/splicProt.sql
  hgsql kgHg17FTemp -e 'load data local infile "splicProt.tab" into table splicProt'

  hgsql kgHg17FTemp -N -e \
  'select varAcc, varProtein.val from sp050415.varProtein,splicProt,proteins050415.spXref3 where accession=parAcc and varProtein.acc=splicProt.varAcc and division="9606"'| \
   awk '{print ">" $1;print $2}' >humanVarProt.fa


   cd /cluster/data/hg17/bed/kgHg17F

# get all Human splicProtBlat records

hgsql hg17 -N -e \
'select splicProtBlat.* from splicProtBlat,proteins050415.spXref3,kgHg17FTemp.splicProt where qName=splicProt.varAcc and parAcc=accession and division="9606"'\
|cut -f 2-22 \
>humanVarProtBlat.psl
  
# Combine the regular protein protBlat records with the variant protein psl records.
  cd /cluster/store10/kg/kgHg17F
  cat ../kgHg17E/protBlat/protBlat.psl humanVarProtBlat.psl >protBlat.psl

  hgLoadPsl hg17 protBlat.psl
# Processing protBlat.psl
# load of protBlat did not go as planned: 104064 record(s), 0 row(s) skipped, 1484 warning(s) loading psl.tab
# Looked into the cause of these 1484 warnings.  It was due to that qBaseInsert and tBaseInsert 
# have negative values, probably due to that this is protein alignment.


# create all_mrna.psl and tight_mrna.psl
   hgsql hg17 -N -e "select * from all_mrna" |cut -f 2-22 >all_mrna.psl

   pslReps -minCover=0.40 -minAli=0.97 -nearTop=0.002 \
           all_mrna.psl tight_mrna.psl /dev/null

# Use overlapSelect to get protein and mRNA alignment overlaps   
   overlapSelect  -statsOutput  -dropped=protOut.psl -overlapThreshold=0.90 \
   -selectFmt=psl -inFmt=psl tight_mrna.psl  protBlat.psl protMrna.stat

   overlapSelect  -mergeOutput  -dropped=protOut.psl -overlapThreshold=0.90 -selectFmt=psl \
   -inFmt=psl tight_mrna.psl  protBlat.psl protMrna.out

# Create protein/mRNA pair and protein lists
   cut -f 10,31 protMrna.out|sort -u >spMrna.tab
   cut -f 10    protMrna.out|sort -u >protein.lis

# Load spMrna.tab into spMrna table in temp DB.
   hgsql kgHg17FTemp < ~/src/hg/lib/spMrna.sql
   hgsql kgHg17FTemp -e 'load data local infile "spMrna.tab" into table spMrna'
   hgsql kgHg17FTemp -e 'create index mrnaID on spMrna(mrnaID)'

# Prepare and perform cluster run of protein/mRNA alignment

# Get mRNA fa file.
   cd /cluster/data/hg17/bed/kgHg17F
   /cluster/data/genbank/bin/i386/gbGetSeqs -native -db=hg17 \
   -gbRoot=/cluster/data/genbank genbank mrna mrna.fa

# Create mrnaSeq table in kgHg17FTemp DB.

   faToTab mrna.fa mrnaSeq.tab

   hgsql kgHg17FTemp -e 'drop table mrnaSeq'
   hgsql kgHg17FTemp <~/src/hg/lib/mrnaSeq.sql
   hgsql kgHg17FTemp -e 'load data local infile "mrnaSeq.tab" into table mrnaSeq'
# Prepare files for cluster run
   ~/src/hg/protein/KG2.sh kgHg17F hg17 050415

# Perform cluster run of protein/mRNA alignment
   ~/src/hg/protein/KG3.sh kgHg17F hg17 050415

# Collect cluster run results
   cd kgBestMrna

   ls out | sed -e 's/prot/do1 prot/g' >doall

# create do1 with the following 2 lines:
   cat << '_EOF_' > do1
echo processing $1
cat out/$1/*.out >>protMrnaRaw.psl
'_EOF_'

   chmod +x do*
   doall

# Filter out low quality alignments
   pslReps -nohead -singleHit -minAli=0.9 protMrnaRaw.psl protMrnaBlat.psl /dev/null
   cut -f 10,14 protMrnaBlat.psl |sort -u >protMrna.lis
   wc protMrna.lis

# Load BLAT results into temp DB.
   hgsql kgHg17FTemp < ~/src/hg/lib/protMrnaBlat.sql
   hgsql kgHg17FTemp -e 'load data local infile "protMrnaBlat.psl" into table protMrnaBlat'
   hgsql kgHg17FTemp -e 'create index tName on protMrnaBlat(tName)'

# Create CDS files from protein/mRNA alignment results.
   hgsql kgHg17FTemp -N -e \
   'select qName,"_",tName,tStart+1,":",tEnd+3 from protMrnaBlat order by qName,tName,tEnd-tStart desc'\
   |sed 's/\t_\t/_/g'|sed 's/\t:\t/../g' >protMrna.cds

# Create protMrna.psl with proteinID_mrnaID as query ID.
   cut -f 22-30 ../protMrna.out > j1.tmp
   cut -f 32-42 ../protMrna.out > j2.tmp
   cut -f 10,31 ../protMrna.out|sed -e 's/\t/_/g' >j3.tmp
   paste j1.tmp j3.tmp j2.tmp >protMrna.psl
   rm j1.tmp j2.tmp j3.tmp

# Run mrnaToGene to create protMrna.gp
   bash
   mrnaToGene -cdsFile=protMrna.cds protMrna.psl protMrna.gp 2>protMrna.err >protMrna.log
   exit

# Prepare refGene and all_mrna gp files.

   cd ..
   hgsql hg17 -N -e 'select * from refGene' >ref.gp

   hgsql hg17 -N -e \
   'select gbCdnaInfo.acc,cds.name from gbCdnaInfo,cds,all_mrna where all_mrna.qName=gbCdnaInfo.acc and   gbCdnaInfo.cds=cds.id' \
   |sort -u > all_mrna.cds

   bash
   mrnaToGene -cdsFile=all_mrna.cds all_mrna.psl all_mrna.gp 2>all_mrna.err > all_mrna.log
   exit

# Align proteins to RefSeq.

   overlapSelect -inCds -statsOutput -overlapThreshold=0.90 -selectFmt=psl -inFmt=genePred\
   protBlat.psl ref.gp ref.stat
   overlapSelect -inCds -dropped=refOut1.gp -overlapThreshold=0.90 -selectFmt=psl -inFmt=genePred\
   protBlat.psl ref.gp protRef.gp

   overlapSelect -mergeOutput -selectCds -dropped=protOut1.psl -overlapThreshold=0.80 -inFmt=psl\
   -selectFmt=genePred ref.gp protBlat.psl protRef.out

   cut -f 10,22 protRef.out | sort -u >spRef.tab
   cut -f 10 protRef.out    | sort -u >protRef.lis

   hgsql kgHg17FTemp -e 'drop table spRef'
   hgsql kgHg17FTemp <~/src/hg/lib/spRef.sql
   hgsql kgHg17FTemp -e 'load data local infile "spRef.tab" into table spRef'

# Prepare and perform cluster runs for protein/RefSeq alignments

   ~/src/hg/protein/KGRef2.sh kgHg17F hg17 050415
   ~/src/hg/protein/KGRef3.sh kgHg17F hg17 050415

   cd kgBestRef
   ls out | sed -e 's/prot/do1 prot/g' >doall

   cat << '_EOF_' > do1
echo processing $1
cat out/$1/*.out >>protRefRaw.psl
'_EOF_'

   chmod +x do*
   doall

# Filter out low quality alignments.
   pslReps -nohead -singleHit -minAli=0.9 protRefRaw.psl protRefBlat.psl /dev/null
   cut -f 10,14 protRefBlat.psl |sort -u >protRef.lis
   wc protRef.lis

   hgsql kgHg17FTemp -e 'drop table protRefBlat'
   hgsql kgHg17FTemp < ~/src/hg/lib/protRefBlat.sql
   hgsql kgHg17FTemp -e 'load data local infile "protRefBlat.psl" into table protRefBlat'
   hgsql kgHg17FTemp -e 'create index tName on protRefBlat(tName)'

# Run gene-check to filter out invalid gp entries
   cd /cluster/data/hg17/bed/kgHg17F
   cat ref.gp kgBestMrna/protMrna.gp all_mrna.gp >kgCandidate0.gp
   gene-check  -incl-ok -ok-genepred-out kgCandidate0.passed.gp -nib-dir /cluster/data/hg17/nib kgCandidate0.gp kgCandidate0.check

   hgsql kgHg17FTemp -e 'drop table kgCandidate0'
   hgsql kgHg17FTemp < ~/src/hg/lib/kgCandidate0.sql 
   hgsql kgHg17FTemp -e  'load data local infile "kgCandidate0.gp" into table kgCandidate0'

   hgsql kgHg17FTemp -e 'drop table geneCheck'
   hgsql kgHg17FTemp < ~/src/hg/lib/geneCheck.sql
   hgsql kgHg17FTemp -e  'load data local infile "kgCandidate0.check" into table geneCheck ignore 2 lines'


# Run kgCheck to get all KG candidates that pass the KG gene check criteria

   kgCheck kgHg17FTemp hg17 kgCandidate0 geneCheck kgCandidate.tab
   hgsql kgHg17FTemp -e  'drop table kgCandidate'
   hgsql kgHg17FTemp < ~/src/hg/lib/kgCandidate.sql
   hgsql kgHg17FTemp -e  'load data local infile "kgCandidate.tab" into table kgCandidate'
   hgsql kgHg17FTemp -e 'create index alignID on kgCandidate(alignID)'

# ####### NEXT TIME AROUND PUT IN AN EXTRA STEP TO BRING IN ITEMS ON A "PUT BACK" LIST
# FOR SPECIAL CASES LIKE SELENOCYSTEINE, NON-AUG INITIATION CODON, RIBOSOMAL SLIPPAGE, ETC.
# #######

# Construct the kgCandidateX table that has alignID in the name field. 
   cut -f 2-10 kgCandidate.tab >j2.tmp
   cut -f 11 kgCandidate.tab >j1.tmp
   paste j1.tmp j2.tmp >kgCandidateX.tab

   hgsql kgHg17FTemp -e  'drop table kgCandidateX'
   hgsql kgHg17FTemp < ~/src/hg/lib/kgCandidateX.sql
   hgsql kgHg17FTemp -e  'load data local infile "kgCandidateX.tab" into table kgCandidateX'

# Score protein/mRna and protein/RefSeq alignments

   kgResultBestMrna2 050415 kgHg17FTemp hg17|sort -u >protMrnaBlatScore.tab
   kgResultBestRef2  050415 kgHg17FTemp hg17|sort -u >protRefScore.tab

# Combine scoring results and load them into temp DB.
   cat protMrnaBlatScore.tab protRefScore.tab >protMrnaScore.tab
   hgsql kgHg17FTemp -e 'drop table protMrnaScore'
   hgsql kgHg17FTemp < ~/src/hg/lib/protMrnaScore.sql
   hgsql kgHg17FTemp -e 'load data local infile "protMrnaScore.tab" into table protMrnaScore'
   hgsql kgHg17FTemp -e 'create index mrnaAcc on protMrnaScore(mrnaAcc)'


# Run kgGetCds to get CDS structure of each gene

   kgGetCds kgHg17FTemp kgCandidateX jY.tmp
   cat jY.tmp |sort -u >kgCandidateY.tab
   rm jY.tmp
   hgsql kgHg17FTemp -e  'drop table kgCandidateY'
   hgsql kgHg17FTemp < ~/src/hg/lib/kgCandidateY.sql
   hgsql kgHg17FTemp -e  'load data local infile "kgCandidateY.tab" into table kgCandidateY'

# Run kgPickPrep to replace long cds structure string with cdsId.
   kgPickPrep kgHg17FTemp kgCandidateZ.tab
   hgsql kgHg17FTemp -e  'drop table kgCandidateZ'
   hgsql kgHg17FTemp < ~/src/hg/lib/kgCandidateZ.sql
   hgsql kgHg17FTemp -e  'load data local infile "kgCandidateZ.tab" into table kgCandidateZ'
   hgsql kgHg17FTemp -e 'create index cdsId on kgCandidateZ(cdsId)'

# Run kgPick to pick the representative a mrna/protein pair for each unique CDS structure.

   kgPick kgHg17FTemp hg17 proteins050415 kg3.tmp dupSpMrna.tmp
   sort -u dupSpMrna.tmp >dupSpMrna.tab

# Sort KG genes to make the kg3.gp table file.
   ~/kent/src/hg/protein/sortKg.pl kg3.tmp >kg3.gp

   hgsql kgHg17FTemp -e  'drop table knownGene'
   hgsql kgHg17FTemp < ~/src/hg/lib/knownGene.sql
   hgsql kgHg17FTemp -e  'load data local infile "kg3.gp" into table knownGene'

   hgsql hg17 -e  'drop table kg3'
   hgsql hg17 < ~/src/hg/lib/kg3.sql
   hgsql hg17 -e  'load data local infile "kg3.gp" into table kg3'

# Perform analysis before renaming the kg3 table to knownGene.

# Load data into hg17 knownGene table.
   hgsql hg17 -e  'drop table knownGene'
   hgsql hg17 < ~/src/hg/lib/knownGene.sql
   hgsql hg17 -e  'load data local infile "kg3.gp" into table knownGene'

# Build knownGeneMrna and knownGenePep tables.

   kgPepMrna kgHg17FTemp hg17 050415
   hgsql hg17 -e  'drop table knownGeneMrna'
   hgsql hg17 < ~/src/hg/lib/knownGeneMrna.sql
   hgsql hg17 -e  'load data local infile "knownGeneMrna.tab" into table knownGeneMrna'
   hgsql hg17 -e  'drop table knownGenePep'
   hgsql hg17 < ~/src/hg/lib/knownGenePep.sql
   hgsql hg17 -e  'load data local infile "knownGenePep.tab" into table knownGenePep'


# Build kgXref table

   kgXref2 kgHg17FTemp 050415 hg17

   hgsql hg17 -e  'drop table kgXref'
   hgsql hg17 < ~/src/hg/lib/kgXref.sql
   hgsql hg17 -e  'load data local infile "kgXref.tab" into table kgXref'

# Build spMrna table

   hgsql hg17 -N -e 'select name, proteinID from knownGene' >kgSpMrna.tab

   hgsql hg17 -e  'drop table spMrna'
   hgsql hg17 <~/src/hg/lib/spMrna.sql
   hgsql hg17 -e 'load data local infile "kgSpMrna.tab" into table spMrna'

# Build kgProtMap table

    ~/src/hg/protein/kgProtMap2.sh kgHg17F hg17 050415

# Update and clean up kgResultBestMrna2.c and then check it in.

#####################################
# Build alias tables.		DONE 5/18/05 Fan.
#	kgAliasM reads from proteins050415.hugo.symbol, proteins050415.hugo.aliases
#	proteins050415.hugo.withdraws, hg17.kgXref.kgID
#	to create kgAliasM.tab and geneAlias.tab
#	by picking out those kgID items from kgXref where
#	kgXref.geneSymbol == hugo.symbol

   kgAliasM hg17 proteins050415

#	kgAliasKgXref reads from hg17.knownGene.proteinID,
#	hg17.knownGene.name, hg17.kgXref.geneSymbol
#	to create kgAliasKgXref.tab

   kgAliasKgXref hg17


#	kgAliasRefseq reads from hg17.knownGene.name,
#	hg17.knownGene.proteinID, hg17.kgXref.refseq
#	to create kgAliasRefseq.tab

   kgAliasRefseq hg17

   hgsql sp050415 -N -e 'select name,gene.val from hg17.knownGene,displayId,gene where displayId.val=proteinID and displayId.acc=gene.acc' \
   | sort -u  > kgAliasP.tab

   hgsql hg17 -N -e 'select name, name from knownGene' >kgAliasDup.tab
   hgsql hg17 -N -e 'select mrnaID, dupMrnaID from dupSpMrna' >>kgAliasDup.tab
   
   cat kgAliasM.tab kgAliasRefseq.tab kgAliasKgXref.tab kgAliasP.tab kgAliasDup.tab| \
   sort |uniq > kgAlias.tab

   hgsql -e "drop table kgAlias;" hg17 
   hgsql hg17 < ~/kent/src/hg/lib/kgAlias.sql
   hgsql hg17 -e 'LOAD DATA local INFILE "kgAlias.tab" into table kgAlias' 

#	kgProtAlias reads from hg17.knownGene.name,
#	hg17.knownGene.proteinID, hg17.knownGene.alignID,
#	proteins050415.spXref3.accession, proteins050415.spSecondaryID, proteins050415.pdbSP.pdb
#	to create kgProtAlias.tab
#

   kgProtAlias hg17 050415

   hgsql hg17 -N -e \
   'select kgID, spDisplayID, protAcc from kgXref where protAcc != ""'\
   | sort -u >kgProtAliasNCBI.tab

# include variant splice protein IDs
   
   hgsql hg17 -N -e \
   'select name, proteinID, parAcc from knownGene,sp050415.varAcc where varAcc=proteinID'\
   |sort -u >kgProtAliasDup.tab
# include duplicate protein IDs from dupSpMrna table
   hgsql hg17 -N -e \
   'select name, knownGene.proteinID, dupProteinID from knownGene, dupSpMrna where name=mrnaID'\
   |sort -u >>kgProtAliasDup.tab

# catch parent acc from dupProteinID too
   hgsql hg17 -N -e\
   'select name, knownGene.proteinID, parAcc from knownGene,dupSpMrna,sp050415.varAcc where name=mrnaID and dupProteinID=varAcc.varAcc'\
   |sort -u >>kgProtAliasDup.tab
    cat kgProtAliasNCBI.tab kgProtAlias.tab kgProtAliasDup.tab | sort -u > kgProtAliasAll.tab

    echo "`date` creating table kgProtAlias"
    hgsql hg17 -e "drop table kgProtAlias;"
    hgsql hg17 <~/src/hg/lib/kgProtAlias.sql; 
    hgsql hg17 -e 'LOAD DATA local INFILE "kgProtAliasAll.tab" into table kgProtAlias;'  

# Build kgSpAlias table

    hgsql hg17 -e \
    'select kgXref.kgID, spID, alias from kgXref, kgAlias where kgXref.kgID=kgAlias.kgID' >j.tmp
    hgsql hg17 -e \
    'select kgXref.kgID, spID, alias from kgXref, kgProtAlias where kgXref.kgID=kgProtAlias.kgID'\
    >>j.tmp
    cat j.tmp|sort -u |grep -v 'kgID' >hg17.kgSpAlias.tab
    rm j.tmp

    hgsql hg17 -e 'drop table kgSpAlias';
    hgsql hg17 < ~/src/hg/lib/kgSpAlias.sql
    hgsql hg17 -e 'load data local infile "hg17.kgSpAlias.tab" into table kgSpAlias'

# MAKE FOLDUTR TABLES (DONE 2005-05-19, Fan)
# First set up directory structure and extract UTR sequence on hgwdev
    ssh hgwdev
    cd /cluster/data/hg17/bed
    mkdir rnaStruct.2005-05-18
    rm rnaStruct
    ln -s rnaStruct.2005-05-18 rnaStruct
    cd rnaStruct
    mkdir -p utr3/split utr5/split utr3/fold utr5/fold
    utrFa hg17 knownGene utr3 utr3/utr.fa
    utrFa hg17 knownGene utr5 utr5/utr.fa

# Split up files and make files that define job.
    ssh kk
    cd /cluster/data/hg17/bed/rnaStruct
    faSplit sequence utr3/utr.fa 50000 utr3/split/s
    faSplit sequence utr5/utr.fa 50000 utr5/split/s
    ls -1 utr3/split > utr3/in.lst
    ls -1 utr5/split > utr5/in.lst
    cd utr3
    cat > gsub <<end
#LOOP
rnaFoldBig split/\$(path1) fold
#ENDLOOP
end
    cp gsub ../utr5

# Do cluster run for 3' UTRs
    gensub2 in.lst single gsub spec
    para create spec
    para try
    para push
# Completed: 35692 of 35692 jobs
# CPU time in finished jobs:    1272085s   21201.42m   353.36h   14.72d  0.040 y
# IO & Wait Time:                102447s    1707.45m    28.46h    1.19d  0.003 y
# Average job time:                  39s       0.64m     0.01h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:            6554s     109.23m     1.82h    0.08d
# Submission to last job:          9100s     151.67m     2.53h    0.11d

# Do cluster run for 5' UTRs 
    cd ../utr5
    gensub2 in.lst single gsub spec
    para create spec
    para try
    para push

# Completed: 33693 of 33693 jobs
# CPU time in finished jobs:     393764s    6562.74m   109.38h    4.56d  0.012 y
# IO & Wait Time:                126205s    2103.41m    35.06h    1.46d  0.004 y
# Average job time:                  15s       0.26m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:           51595s     859.92m    14.33h    0.60d
# Submission to last job:         52057s     867.62m    14.46h    0.60d

# Load database
    ssh hgwdev
    cd /cluster/data/hg17/bed/rnaStruct/utr5
    hgLoadRnaFold hg17 foldUtr5 fold
    cd ../utr3
    hgLoadRnaFold hg17 foldUtr3 fold

# Clean up
    rm -r split fold err batch.bak
    cd ../utr5
    rm -r split fold err batch.bak

# Build KEGG pathway tables.  DONE 5/19/05.  Fan.
   ssh hgwdev
   cd /cluster/store10/kg/kgHg17F
   md kegg
   cd kegg

   ~/src/hg/protein/KGpath.sh kgHg17F hg17 050415

   hgsql hg17 -e "drop table keggMapDesc"
   hgsql hg17 -e "drop table keggPathway"
   hgsql hg17 <~/src/hg/lib/keggMapDesc.sql
   hgsql hg17 <~/src/hg/lib/keggPathway.sql
   hgsql hg17 -e 'load data local infile "keggMapDesc.tab" into table keggMapDesc'
   hgsql hg17 -e 'load data local infile "keggPathway.tab" into table keggPathway'
  
# Build CGAP pathway tables
# RELOAD cgapAlias TABLE AFTER REMOVING REPLICATE ROWS (hartera, 2005-07-26)
# duplicate rows. (hartera, 2005-07-26)
# RELOADED cgapAlias AGAIN AS TOO MANY ROWS REMOVED BEFORE (hartera, 2005-10-06)

   cd ..
   ~/src/hg/protein/KGcgap.sh kgHg17F hg17 050415
   hgsql hg17 -e "drop table cgapAlias"
   hgsql hg17 -e "drop table cgapBiocDesc"
   hgsql hg17 -e "drop table cgapBiocPathway"
   hgsql hg17 <~/src/hg/lib/cgapAlias.sql
   hgsql hg17 <~/src/hg/lib/cgapBiocDesc.sql
   hgsql hg17 <~/src/hg/lib/cgapBiocPathway.sql
   hgsql hg17 -e 'load data local infile "cgapAlias.tab" \
                 into table cgapAlias'
   hgsql hg17 -e 'load data local infile "cgapBIOCARTAdescSorted.tab" into table cgapBiocDesc'
   hgsql hg17 -e 'load data local infile "cgapBIOCARTA.tab" into table cgapBiocPathway'

   # RELOAD cgapAlias TABLE. Sort and reload alias tab file to remove 
   # duplicate rows. (hartera, 2005-07-26)
   # DO TABLE RELOAD AGAIN AS sort -nu REMOVES MORE ROWS THAN sort -u
   # OR sort -n | uniq.
   #USE sort -n then uniq TO SORT ON THE IDs AND THEN UNIQ(hartera, 2005-10-06)
   cd /cluster/store10/kg/kgHg17F
   hgsql hg17 -e "drop table cgapAlias"
   # cgapAlias.tab has replicated rows so sort and unique before loading
   sort -n cgapAlias.tab | uniq > cgapAliasSorted.tab
   hgsql hg17 < ~/kent/src/hg/lib/cgapAlias.sql 
   hgsql hg17 -e 'load data local infile "cgapAliasSorted.tab" \
                 into table cgapAlias'

# LOAD ENSEMBL GENES (DONE, 5/23/05, Fan)
# Ensembl changed things again! Please note there are two subtle changes to make it work.

    mkdir /cluster/data/hg17/bed/ensembl
    cd /cluster/data/hg17/bed/ensembl
    mkdir new
    cd new

    # Get the ensembl protein data from 
    # http://www.ensembl.org/Homo_sapiens/martview
    # Follow this sequence through the pages:
    # Page 1) Make sure that the Homo_sapiens choice is selected. Hit next.
    # Page 2) Uncheck the "Limit to" box in the region choice. Then hit next.
    # Page 3) Choose the "Structures" box. 
    # Page 4) Choose GTF as the ouput.  choose gzip compression.  hit export.
    # Save as ensemblGene.gtf.gz

    # This time, there are some extra lines, like '		1;', 
    # that are causing problems, so added an extra filter in the beginning 
    # to get rid of them.
    # Ensembl handles random chromosomes differently than us, so we
    # strip this data.  Fortunately it just loses a couple of genes.
    # Add "chr" to front of each line in the gene data gtf file to make 
    # it compatible with our software.
    # Finally, get rid of the ".1" or ".2" after the name

cat ensemblGene.gtf |sed -e 's/\t\t/xxxxx/g' \
    |grep -v xxxxx \
    | grep -v ^6_DR51 \
    | grep -v ^DR51 \
    | grep -v ^DR52 \
    | grep -v ^DR53 \
    | grep -v _NT_ \
    | perl -wpe 's/^([0-9]|X|Y|Un|MT)/chr$1/ \
                 || die "Line $. doesnt start with human chrom:\n$_"' \
    | sed -e 's/chrMT/chrM/g' \
    | sed -e 's/\..\"/\"/g' \
>ensGene.gtf

    ssh hgwdev
    cd /cluster/data/hg17/bed/ensembl/new
    /cluster/bin/i386/ldHgGene hg17 ensGene ensGene.gtf
# Read 33581 transcripts in 699580 lines in 1 files
# 33581 groups 25 seqs 1 sources 4 feature types
# 33581 gene predictions

    # ensGtp associates geneId/transcriptId/proteinId for hgPepPred and 
    # hgKnownToSuper.  Use ensMart to create it as above, except:
    # Page 3) Choose the "Features" box. In "Ensembl Attributes", check 
    # Ensembl Gene ID, Ensembl Transcript ID, Ensembl Peptide ID.  
    # Choose Text, tab-separated as the output format.  Result name ensGtp.
    # Save file as ensGtp.txt.gz
    gunzip ensGtp.txt.gz
    hgsql hg17 -e 'drop table ensGtp'
    hgsql hg17 < ~/kent/src/hg/lib/ensGtp.sql
    hgsql hg17 -e 'load data local infile "ensGtp.txt" into table ensGtp ignore 1 lines'

# ensMart has some problem with the resulting ensemblPep.fa.gz, so use different
# processing step instead:

    wget -timestamp \
    ftp://ftp.ensembl.org/pub/current_human/data/fasta/pep/Homo_sapiens.NCBI35.may.pep.fa.gz
    zcat Homo_sapiens.NCBI35.may.pep.fa.gz | sed -e "s/transcript:/\n>/g" | grep -v 'gene:' >ensPep.fa
    faToTab -type=protein ensPep.fa ensPep.tab
    hgsql hg17 -e 'drop table ensPep'
    hgsql hg17 < ~/kent/src/hg/lib/ensPep.sql
    hgsql hg17 -e 'load data local infile "ensPep.tab" into table ensPep'

# kept the following, just in case Ensembl fixed the problem in the future
    # Load Ensembl peptides:
    # Get them from ensembl as above in the gene section except for
    # Page 3) Choose the "Sequences" box. 
    # Page 4) Transcripts/Proteins.  Peptide.  Format = FASTA.
    # Save file as ensemblPep.fa.gz
# gunzip ensemblPep.fa.gz
# hgPepPred hg17 ensembl ensemblPep.fa

# UPDATE GENE SORTER TABLES (AKA: FAMILY BROWSER) (STARTED - 2005-05-21, DONE 2005-05-23 - Fan)
#	This should be done after knownGene tables are complete from known gene
#	process.
#
# Cluster together various alt-splicing isoforms.
#	Creates the knownIsoforms and knownCanonical tables
ssh hgwdev
mkdir /cluster/data/hg17/bed/geneSorter.2005-05-21
# remove old symbolic link
rm /cluster/data/hg17/bed/geneSorter
ln -s /cluster/data/hg17/bed/geneSorter.2005-05-21 \
	/cluster/data/hg17/bed/geneSorter
cd /cluster/data/hg17/bed/geneSorter
hgClusterGenes hg17 knownGene knownIsoforms knownCanonical

# Extract peptides from knownGenes into fasta file
# and create a blast database out of them.
mkdir /cluster/data/hg17/bed/geneSorter/blastp
cd /cluster/data/hg17/bed/geneSorter/blastp
pepPredToFa hg17 knownGenePep known.faa
#	You may need to build this binary in src/hg/near/pepPredToFa
/scratch/blast/formatdb -i known.faa -t known -n known
#	This command is in /projects/compbio/bin/$MACH/formatdb

# Copy over database to bluearc
rm -fr /cluster/bluearc/hg17/blastp
mkdir -p /cluster/bluearc/hg17/blastp
cp -p /cluster/data/hg17/bed/geneSorter/blastp/known.* \
	/cluster/bluearc/hg17/blastp

# Split up fasta file into bite sized chunks for cluster
cd /cluster/data/hg17/bed/geneSorter/blastp
mkdir split
faSplit sequence known.faa 8000 split/kg

# Make parasol run directory
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/self
cd /cluster/data/hg17/bed/geneSorter/blastp/self
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
-p blastp -d /cluster/bluearc/hg17/blastp/known -i $1 -o $2 \
-e 0.01 -m 8 -b 1000
'_EOF_'
    # << keep emacs happy
chmod +x blastSome

# Make gensub2 file
cat  << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#	'ls ../../split/*.fa' is too much, hence the echo
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
# Wait a couple of minutes, and do a para check,  if all is good
# then do a
para push
# This should finish in ~15 minutes if the cluster is free.
# Completed: 7739 of 7739 jobs
# CPU time in finished jobs:     150459s    2507.64m    41.79h    1.74d  0.005 y
# IO & Wait Time:                 22325s     372.09m     6.20h    0.26d  0.001 y
# Average job time:                  22s       0.37m     0.01h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:             198s       3.30m     0.06h    0.00d
# Submission to last job:          2019s      33.65m     0.56h    0.02d

# Load into database.  This takes about 30 minutes
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/self/run/out
time hgLoadBlastTab hg17 knownBlastTab *.tab
# Scanning through 7739 files
# Loading database with 9836439 rows
# 232.300u 42.580s 23:13.41 19.7% 0+0k 0+0io 205pf+0w

cd /cluster/data/hg17/bed/geneSorter
# Create table that maps between known genes and RefSeq
hgMapToGene hg17 refGene knownGene knownToRefSeq
#	may need to build this command in src/hg/near/hgMapToGene
#	hgsql -e "select count(*) from knownToRefSeq;" hg17
#	row count changed 34667

# Create table that maps between known genes and LocusLink
hgsql --skip-column-names -e "select mrnaAcc,locusLinkId from refLink" hg17 \
	> refToLl.txt
hgMapToGene hg17 refGene knownGene knownToLocusLink -lookup=refToLl.txt
#	hgsql -e "select count(*) from knownToLocusLink;" hg17
#	row count changed to 34667

# Create table that maps between known genes and Pfam domains
hgMapViaSwissProt hg17 knownGene name proteinID Pfam knownToPfam
#	hgsql -e "select count(*) from knownToPfam;" hg17
#	row count changed to 36010

# Create table to map between known genes and GNF Atlas2
# expression data.
    hgMapToGene hg17 gnfAtlas2 knownGene knownToGnfAtlas2 '-type=bed 12'
#	hgsql -e "select count(*) from knownToGnfAtlas2;" hg17
#	row count changed to 32381

# Create expression distance table - takes about an hour
    hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
    	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
	-lookup=knownToGnfAtlas2 &
# Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
# Got 32381 unique elements in hgFixed.gnfHumanAtlas2MedianRatio

#	hgsql -e "select count(*) from gnfAtlas2Distance;" hg17
#	row count changed to 32381000

# Create a table that maps between known genes and 
# the nice affy expression data.
hgMapToGene "-type=bed 12" hg17 affyUclaNorm knownGene knownToU133
#	hgsql -e "select count(*) from knownToU133;" hg17
#	row count changed to 32886

# Create expression distance table.  This will take about 2.5 hours
cd /tmp
cp -p ~/kent/src/hg/near/hgExpDistance/affyUcla.weight .
time hgExpDistance hg17 affyUclaNorm affyUclaExp knownExpDistance \
	-weights=affyUcla.weight -lookup=knownToU133 &
# Have 43039 elements in affyUclaNorm
# 211 genes, 42 weights, 26.500000 total wieght
# Got 32886 unique elements in affyUclaNorm

# Create table that maps between known genes and 
# the GNF data.
cd /tmp
hgMapToGene hg17 affyU95 knownGene knownToU95
#	row count changed to 17501
#	hgFixed.gnfHumanU95Exps argument is unused, no need to exist
hgExpDistance hg17 hgFixed.gnfHumanU95MedianRatio \
	hgFixed.gnfHumanU95Exps gnfU95Distance  -lookup=knownToU95 &
# Have 11545 elements in hgFixed.gnfHumanU95MedianRatio
# Got 16450 unique elements in hgFixed.gnfHumanU95MedianRatio
#	row count changed to 16450000  

# Create known gene mapping table and expression distance tables
# for GNF Atlas 2.  (The hgExpDistance takes only 10 minutes.)

hgMapToGene hg17 affyGnf1h knownGene knownToGnf1h
hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
	-lookup=knownToGnf1h &
# Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
# Got 8814 unique elements in hgFixed.gnfHumanAtlas2MedianRatio

cd /cluster/data/hg17/bed/geneSorter
hgMapToGene hg17 affyU133Plus2 knownGene knownToU133Plus2
#	row count changed to 35055 

#### UPDATE GO DATABASE (DONE 5/21/05 Fan)

# Download the terms and make the database.
ssh hgwdev
mkdir /cluster/store1/geneOntology/20050521
cd /cluster/store1/geneOntology/20050521

wget --timestamping http://www.godatabase.org/dev/database/archive/latest/go_200504-assocdb-data.gz

hgsql mysql <<end
create database go050521;
end
zcat go_*data.gz | sed -e 's/ENGINE=MyISAM DEFAULT CHARSET=latin1/TYPE=MyISAM/g' >j.tmp
hgsql go050521 <j.tmp
rm j.tmp

wget --timestamping ftp://ftp.geneontology.org/pub/go/gene-associations/gene_association.goa_uniprot.gz

zcat gene_association.goa_uniprot.gz | hgGoAssociation go050521 goaPart stdin 
# Passed 5589891 of 6584507 of 6584507, 84.89%

# Ask sys-admin to switch the database pointer go to point to go050521.

cd /cluster/data/hg17/bed/geneSorter

# Rebuilt Ensembl Gene tables.  See documentation (5/23/05 Fan) above.

# Create knownToEnsembl column
hgMapToGene hg17 ensGene knownGene knownToEnsembl
#	table row count went from previous version: 38251 to 35436

# Make knownToCdsSnp table 
  ssh hgwdev
  nice hgMapToGene hg17 snp knownGene knownToCdsSnp -all -cds
# row count 94394
# approx. 5 minutes running time

# C.ELEGANS BLASTP FOR GENE SORTER 
    # Make C. elegans ortholog column using blastp on wormpep.
    # First make C. elegans protein database and copy it to iscratch/i
    # if it doesn't exist already:

# The following section is done during mm6 build already.
#    ssh eieio
#    mkdir /cluster/data/ce2/bed/blastp
#    cd /cluster/data/ce2/bed/blastp
#    # Point a web browser at ftp://ftp.sanger.ac.uk/pub/databases/wormpep/
#    # to find out the latest version.  Then use that in place of 142 below.
#    wget -O wormPep142.faa ftp://ftp.sanger.ac.uk/pub/databases/wormpep/wormpep142/wormpep142
#    formatdb -i wormPep142.faa -t wormPep142 -n wormPep142
#    ssh kkr1u00
#    if (-e /iscratch/i/ce2/blastp) then
#      rm -r /iscratch/i/ce2/blastp
#    endif
#    mkdir -p /iscratch/i/ce2/blastp
#    cp /cluster/data/ce2/bed/blastp/wormPep142.p?? /iscratch/i/ce2/blastp
#    iSync

    # Make parasol run directory 
    ssh kk
    mkdir -p /cluster/data/hg17/bed/blastp/ce2/run/out
    cd /cluster/data/hg17/bed/blastp/ce2/run
    # Make blast script
    cat > blastSome <<end
#!/bin/csh
setenv BLASTMAT /iscratch/i/blast/data
/iscratch/i/blast/blastall -p blastp -d /iscratch/i/ce2/blastp/wormPep142 -i \$1 -o \$2 -e 0.01 -m 8 -b 1
end
    chmod a+x blastSome
    # Make gensub2 file
    cat > gsub <<end
#LOOP
blastSome {check in line+ \$(path1)} {check out line out/\$(root1).tab}
#ENDLOOP
end
    # Create parasol batch
    ls -1S /cluster/data/hg17/bed/geneSorter/blastp/split >split.lst
    #EDIT split.lst to add "../../../geneSorter/blastp/split/" in front of "kg"
    gensub2 split.lst single gsub spec
    para create spec
    para try, check, push, check, ...
# Completed: 7739 of 7739 jobs
# CPU time in finished jobs:      60973s    1016.22m    16.94h    0.71d  0.002 y
# IO & Wait Time:                 21292s     354.86m     5.91h    0.25d  0.001 y
# Average job time:                  11s       0.18m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              50s       0.83m     0.01h    0.00d
# Submission to last job:           570s       9.50m     0.16h    0.01d

# Load into database.  
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastp/ce2/run/out
    hgLoadBlastTab hg17 ceBlastTab -maxPer=1 *.tab
# Scanning through 7739 files
# Loading database with 25706 rows

# Make mouse ortholog column using blastp on mouse known genes.
# First make mouse protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This already exists.  See makeMm6.doc for procedure
#	the directory: /cluster/bluearc/scratch/mus/mm6/blastp should have data

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/mm6
cd /cluster/data/hg17/bed/geneSorter/blastp/mm6
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
-p blastp -d /cluster/panasas/home/store/mm6/blastp/known \
-i $1 -o $2 -e 0.001 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#	this echo trick is used because otherwise the command line is
#	too long and you can not do a simple ls
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...

# Completed: 7739 of 7739 jobs
# CPU time in finished jobs:      65337s    1088.95m    18.15h    0.76d  0.002 y
# IO & Wait Time:                 20794s     346.56m     5.78h    0.24d  0.001 y
# Average job time:                  11s       0.19m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              80s       1.33m     0.02h    0.00d
# Submission to last job:           598s       9.97m     0.17h    0.01d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/mm6/run/out
hgLoadBlastTab hg17 mmBlastTab -maxPer=1 *.tab
# Scanning through 7739 files 
#	row count changed to 32880  

# Make rat ortholog column using blastp on rat known genes.
# First make rat protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This already exists.  See makeRn3.doc for procedure.
#	Files were put in this directory: /cluster/bluearc/rn3/blastp/

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/rn3
cd /cluster/data/hg17/bed/geneSorter/blastp/rn3
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
-p blastp -d /cluster/bluearc/rn3/blastp/known \
-i $1 -o $2 -e 0.001 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#	this echo trick is used because otherwise the command line is
#	too long and you can not do a simple ls
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7739 of 7739 jobs
# CPU time in finished jobs:      28325s     472.08m     7.87h    0.33d  0.001 y
# IO & Wait Time:                 20416s     340.27m     5.67h    0.24d  0.001 y
# Average job time:                   6s       0.10m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              24s       0.40m     0.01h    0.00d
# Submission to last job:           617s      10.28m     0.17h    0.01d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/rn3/run/out
hgLoadBlastTab hg17 rnBlastTab -maxPer=1 *.tab
# Scanning through 7739 files
# Loading database with 24140 rows

# ZEBRAFISH BLASTP FOR GENE SORTER 
    # Make Danio rerio (zebrafish) ortholog column using blastp on Ensembl.
    # First make protein database and copy it to iscratch/i
    # if it doesn't exist already:
    ssh kkstore
    mkdir /cluster/data/danRer2/bed/blastp
    cd /cluster/data/danRer2/bed/blastp
    wget ftp://ftp.ensembl.org/pub/current_zebrafish/data/fasta/pep/Danio_rerio.ZFISH4.may.pep.fa.gz 
    zcat Dan*.pep.fa.gz > ensembl.faa
    /scratch/blast/formatdb -i ensembl.faa -t ensembl -n ensembl
    ssh kkr1u00
    if (-e /iscratch/i/danRer2/blastp) then
      rm -r /iscratch/i/danRer2/blastp
    endif
    mkdir -p /iscratch/i/danRer2/blastp
    cp /cluster/data/danRer2/bed/blastp/ensembl.p?? /iscratch/i/danRer2/blastp
    iSync

    # Make parasol run directory 
    ssh kk
    mkdir -p /cluster/data/hg17/bed/blastp/danRer2/run/out
    cd /cluster/data/hg17/bed/blastp/danRer2/run
    # Make blast script
    cat > blastSome <<end
#!/bin/csh
setenv BLASTMAT /iscratch/i/blast/data
/iscratch/i/blast/blastall -p blastp -d /iscratch/i/danRer2/blastp/ensembl -i \$1 -o \$2 -e 0.005 -m 8 -b 1
end
    chmod a+x blastSome
    # Make gensub2 file
    cat > gsub <<end
#LOOP
blastSome {check in line+ \$(path1)} {check out line out/\$(root1).tab}
#ENDLOOP
end
    # Create parasol batch
    ls -1S /cluster/data/hg17/bed/geneSorter/blastp/split \
    |sed -e 's=kg=../../../geneSorter/blastp/split/kg=g' > split.lst
    gensub2 split.lst single gsub spec
    para create spec
    para try, check, push, check, ...
# Completed: 7739 of 7739 jobs
# CPU time in finished jobs:     113595s    1893.26m    31.55h    1.31d  0.004 y
# IO & Wait Time:                 26231s     437.18m     7.29h    0.30d  0.001 y
# Average job time:                  18s       0.30m     0.01h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              99s       1.65m     0.03h    0.00d
# Submission to last job:           445s       7.42m     0.12h    0.01d

    # Load into database.  
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastp/danRer2/run/out
    hgLoadBlastTab hg17 drBlastTab -maxPer=1 *.tab
# Scanning through 7739 files
# Loading database with 30731 rows


# Make Saccharomyces cerevisiae (yeast) ortholog column using blastp on RefSeq.
# First make protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This is already done, see makeMm3.doc for procedure
#	the directory: /cluster/bluearc/sc1/blastp should have data

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/sc1
cd /cluster/data/hg17/bed/geneSorter/blastp/sc1
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
-p blastp -d /cluster/bluearc/sc1/blastp/sgd \
-i $1 -o $2 -e 0.01 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ... 
# Completed: 7739 of 7739 jobs
# CPU time in finished jobs:      18630s     310.50m     5.17h    0.22d  0.001 y
# IO & Wait Time:                 20776s     346.27m     5.77h    0.24d  0.001 y
# Average job time:                   5s       0.08m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              15s       0.25m     0.00h    0.00d
# Submission to last job:           295s       4.92m     0.08h    0.00d
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/sc1/run/out
hgLoadBlastTab hg17 scBlastTab -maxPer=1 *.tab
# Loading database with 16540 rows

# Make Drosophila melanagaster ortholog column using blastp on FlyBase.
# First make SwissProt protein database and copy it to cluster/bluearc
# The following section was already done.
# cd /cluster/data/dm1/bed
# mkdir blastp
# cd blastp
#wget ftp://ftp.fruitfly.org/pub/download/dmel_RELEASE3-1/FASTA/whole_genome_translation_dmel_RELEASE3-1.FASTA.gz
# zcat whole_ge*.gz | faFlyBaseToUcsc stdin flyBase.faa
# formatdb -i flyBase.faa -t flyBase -n flyBase
# if (-e /cluster/bluearc/dm1/blastp) then
#    rm -r /cluster/bluearc/dm1/blastp
# endif
# mkdir -p /cluster/bluearc/dm1/blastp
# cp /cluster/data/dm1/bed/blastp/flyBase.p?? /cluster/bluearc/dm1/blastp

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/dm1
cd /cluster/data/hg17/bed/geneSorter/blastp/dm1
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
-p blastp -d /cluster/bluearc/dm1/blastp/flyBase \
-i $1 -o $2 -e 0.01 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7739 of 7739 jobs
# CPU time in finished jobs:      73518s    1225.30m    20.42h    0.85d  0.002 y
# IO & Wait Time:                 45038s     750.63m    12.51h    0.52d  0.001 y
# Average job time:                  15s       0.26m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              69s       1.15m     0.02h    0.00d
# Submission to last job:           762s      12.70m     0.21h    0.01d 

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/dm1/run/out
hgLoadBlastTab hg17 dmBlastTab -maxPer=1 *.tab
# Loading database with 27212 rows

# update knownToHInv table
# Verified that there is now new release of HInv data.
hgMapToGene hg17 HInvGeneMrna knownGene knownToHInv
# count changed to 28851

# The new KG process no longer need entries in knownGeneLink (used to store
# info for DNA based RefSeqs.  So clean out the old data in knownGeneLink.

hgsql hg17 -e "delete from knownGeneLink"

#### RE-BUILD SUPERFAMILY RELATED TABLES (DONE - 2005-05-27 - Fan)

# Download latest Superfamily data files and build the Superfamily DB
# from supfam.mrc-lmb.cam.ac.uk

    mkdir /cluster/store10/superfamily/050524
    ln -s /cluster/store10/superfamily/050524 /cluster/data/superfamily/050524
    cd /cluster/data/superfamily/050524

# ftp over the following two files:
ass_22-May-2005.tab.gz
supfam_22-May-2005.sql.gz
    gzip -d *.gz

# Load the Superfamily database
    hgsql hg17 -e "create database superfam050524"
    nice hgsql superfam050524 < supfam_22-May-2005.sql &

# This may take about an hour.

# Make sure to add an index on id of the des table of superfam050524.
    hgsql superfam050524 -e "create index id on des(id);"

    hgsql superfam050524 < ~/src/hg/lib/sfAssign.sql
    hgsql superfam050524 -e 'load data local infile "ass_22-May-2005.tab" into table 
superfam050524.sfAssign;'

# Build or rebuild Superfamily track and create sf tables needed for PB

   hgsql hg17 < ~/src/hg/lib/sfAssign.sql

   cd /cluster/data/superfamily/050524  
   hgsql hg17 -e 'load data local infile "ass_22-May-2005.tab" into table hg17.sfAssign;'

# If hg17.sfDes already exists, drop it.

   hgsql superfam050524 -N -e "select * from des" >sfDes.tab
   hgsql hg17 < ~/src/hg/lib/sfDes.sql
   hgsql hg17 -e 'load data local infile "sfDes.tab" into table sfDes'

# If hg17.superfamily already exists, drop it.
   cd /cluster/data/hg17/bed
   mkdir /cluster/data/hg17/sf.2004-1128
   ln -s sf.2004-1128 sf
   hgSuperfam hg17 > sf.log

# It is normal that many proteins does not have corresponding Superfamily entries.

# If hg17.sfDescription exists, drop it.

   hgsql hg17 < ~/src/hg/lib/sfDescription.sql
   hgsql hg17 -e 'LOAD DATA local INFILE "sfDescription.tab"  into table hg17.sfDescription;'

# Finally, load the superfamily table.

   hgLoadBed hg17 superfamily superfamily.tab -tab

# Create knownToSuperfamily table
# Note hs is changed into ht for this Superfamily release.
   
   cat /cluster/data/superfamily/050524/ass_22-May-2005.tab \
   | hgKnownToSuper hg17 hs stdin
# created 25287 rows in knownToSuper

# Build tables needed by pbGlobal in proteins050415

   cd /cluster/data/superfamily/050524  
   hgsql proteins050415 -e 'load data local infile "ass_22-May-2005.tab" into table sfAssign'

   hgsql proteins050415 -e 'load data local infile "sfDes.tab" into table sfDes'

   cd /cluster/store10/kg/kgHg17F
   hgsql proteins050415 -e 'load data local infile "ensemblXref.tab" into table ensemblXref'

# These sf tables and ensemblXref3 are needed for non-HMR KG proteins.
# Should add content of ensemblXref3 of mm6 after it is done.
# And similarly for rn4 and possibly for other non-HMR species.
# CCDS <-> knownGene mapping need to be updated (markd 2005-05-29)
# this should be part of the known gene build
    /cluster/data/genbank/bin/i386/mkCcdsGeneMap -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap

# AUGUSTUS GENES (DONE 6/1/2005 Andy)
    ssh hgwdev
    cd /cluster/data/hg17/bed
    mkdir augustus
    cd augustus/
    wget http://augustus.gobics.de/predictions/hg17/hg17.allchr.augustus.gtf.gz
    cp /cluster/data/dm2/bed/augustus/cleanAugustus.awk .
    zcat hg17.allchr.augustus.gtf.gz | awk -f cleanAugustus.awk | gzip > hg17.allchr.augustus.clean.gtf.gz
    ldHgGene -gtf hg17 augustus hg17.allchr.augustus.clean.gtf.gz
    rm hg17.allchr.augustus.gtf.gz

# MAKE Mouse Proteins track (DONE for chr13 braney ~5/25/05)
    ssh kkstore01
    mkdir -p /cluster/data/hg17/blastDb
    cd /cluster/data/hg17/blastDb
    awk "{print \$2}" ../*/chr*/*.lft > subChr.lst
    for i in `cat subChr.lst`
    do
	ln -s ../*/chr*/$i.fa 
	echo formatdb -i $i.fa -p F
	formatdb -i $i.fa -p F
    done
    rm *.log *.fa list
    cd ..
    for i in `cat chrom.lst`; do cat  $i/chr*/*.lft  ; done > jkStuff/subChr.lft

    ssh kkr1u00
    rm -rf /iscratch/i/hg17/blastDb
    mkdir -p /iscratch/i/hg17/blastDb
    cd /cluster/data/hg17/blastDb
    for i in nhr nin nsq; do cp *.$i /iscratch/i/hg17/blastDb     ; echo $i; done

    cd
    iSync > sync.out

    mkdir -p /cluster/data/hg17/bed/tblastn.mm6KG
    cd /cluster/data/hg17/bed/tblastn.mm6KG
    echo  /panasas/store/hg17/blastDb/*.nsq  | xargs ls -S | sed "s/\.nsq//"  > query.lst    
    # back to kkstore01
    exit

    cd /cluster/data/hg17/bed/tblastn.mm6KG
    rm -rf /cluster/bluearc/hg17/bed/tblastn.mm6KG/kgfa
    mkdir -p /cluster/bluearc/hg17/bed/tblastn.mm6KG/kgfa
    split -l 560 /cluster/data/mm6/bed/blat.mm6KG/mm6KG.psl /cluster/bluearc/hg17/bed/tblastn.mm6KG/kgfa/kg
    ln -s /cluster/bluearc/hg17/bed/tblastn.mm6KG/kgfa kgfa
    cd kgfa
    for i in *; do pslxToFa $i $i.fa; rm $i; done
    cd ..
    ls -1S kgfa/*.fa > kg.lst
    rm -rf /cluster/bluearc/hg17/bed/tblastn.mm6KG/blastOut
    mkdir -p /cluster/bluearc/hg17/bed/tblastn.mm6KG/blastOut
    ln -s /cluster/bluearc/hg17/bed/tblastn.mm6KG/blastOut
    for i in `cat kg.lst`; do  mkdir blastOut/`basename $i .fa`; done
    tcsh
    cat << '_EOF_' > blastGsub
#LOOP
blastSome $(path1) {check in line $(path2)} {check out exists blastOut/$(root2)/q.$(root1).psl } 
#ENDLOOP
'_EOF_'
    cat << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/iscratch/i/blast/data
export BLASTMAT
g=`basename $2`
f=/tmp/`basename $3`.$g
for eVal in 0.01 0.001 0.0001 0.00001 0.000001 1E-09 1E-11
do
if /scratch/blast/blastall -M BLOSUM80 -m 0 -F no -e $eVal -p tblastn -d $1 -i $2 -o $f.8
then
        mv $f.8 $f.1
        break;
fi
done
if test -f  $f.1
then
    if /cluster/bin/i386/blastToPsl $f.1 $f.2
    then
	liftUp -nosort -type=".psl" -nohead $f.3 ../../jkStuff/subLiftAll.lft carry $f.2                     
	liftUp -nosort -type=".psl" -nohead $f.4 ../../jkStuff/liftAll.lft carry $f.3                     
	liftUp -nosort -type=".psl" -pslQ -nohead $3.tmp /cluster/data/mm6/bed/blat.mm6KG/protein.lft warn $f.4       

        if pslCheck -prot $3.tmp                                                                          
        then                                                                                              
            mv $3.tmp $3                                                                                  
            rm -f $f.1 $f.2 $f.3  $f.4
        fi
        exit 0                                                                                            
    fi                                                                                                    
fi                                                                                                        
rm -f $f.1 $f.2 $3.tmp $f.8 $f.3 $f.4
exit 1
'_EOF_'

    chmod +x blastSome
    gensub2 query.lst kg.lst blastGsub blastSpec

    ssh kk
    cd /cluster/data/hg17/bed/tblastn.mm6KG
    para create blastSpec
    para push
# Completed: 214524 of 214524 jobs
# CPU time in finished jobs:   44907411s  748456.85m 12474.28h  519.76d  1.424 y
# IO & Wait Time:                712709s   11878.48m   197.97h    8.25d  0.023 y
# Average job time:                 213s       3.54m     0.06h    0.00d
# Longest finished job:            1363s      22.72m     0.38h    0.02d
# Submission to last job:         75910s    1265.17m    21.09h    0.88d

# just for chr13
# completed: 55290 of 55290 jobs
# cCPU time in finished jobs:    1487547s   24792.46m   413.21h   17.22d  0.047 y
# cIO & Wait Time:                148854s    2480.89m    41.35h    1.72d  0.005 y
# cAverage job time:                  30s       0.49m     0.01h    0.00d
# cLongest running job:                0s       0.00m     0.00h    0.00d
# cLongest finished job:              98s       1.63m     0.03h    0.00d
# cSubmission to last job:          3904s      65.07m     1.08h    0.05d


    cat << '_EOF_' > chainGsub
#LOOP
chainSome $(path1)
#ENDLOOP
'_EOF_'

    ssh kki
    cd /cluster/data/hg17/bed/tblastn.mm6KG
    tcsh
    cat << '_EOF_' > chainOne
(cd $1; cat q."$2"* | simpleChain -prot -outPsl -maxGap=200000 stdin ../c.`basename $1`.$2.psl)
'_EOF_'
    chmod +x chainOne

    for j in blastOut/kg??; do for i in `cat ../../chrom.lst`; do echo chainOne $j chr"$i"; done ; done > chainSpec

    para create chainSpec
    para push
# CPU time in finished jobs:         90s       1.50m     0.03h    0.00d  0.000 y
# IO & Wait Time:                 19151s     319.18m     5.32h    0.22d  0.001 y
# Average job time:                   3s       0.04m     0.00h    0.00d
# Longest finished job:               5s       0.08m     0.00h    0.00d
# Submission to last job:          1642s      27.37m     0.46h    0.02d

# Completed: 7695 of 7695 jobs
# CPU time in finished jobs:         48s       0.80m     0.01h    0.00d  0.000 y
# IO & Wait Time:                 18931s     315.51m     5.26h    0.22d  0.001 y
# Average job time:                   2s       0.04m     0.00h    0.00d
# Longest finished job:               6s       0.10m     0.00h    0.00d
# Submission to last job:          1618s      26.97m     0.45h    0.02d

    exit
    # back to kkstore01
    cd /cluster/data/hg17/bed/tblastn.mm6KG/blastOut
    for i in kg??
    do 
	cat c.$i.*.psl | awk "(\$13 - \$12)/\$11 > 0.6 {print}" > c60.$i.psl
	sort -rn c60.$i.psl | pslUniq stdin u.$i.psl
	awk "((\$1 / \$11) ) > 0.90 { print   }" c60.$i.psl > m60.$i.psl
	echo $i
    done

    cat u.*.psl m60* | sort -T /tmp -k 14,14 -k 17,17n -k 17,17n  | uniq  > /cluster/data/hg17/bed/tblastn.mm6KG/blastMm6KG.psl
    cd ..
    ssh hgwdev
    cd /cluster/data/hg17/bed/tblastn.mm6KG
    hgLoadPsl hg17 blastHg17KG.psl
    # 1425966 bases of 64944656 (2.196%) 

    # back to kkstore01
    rm -rf blastOut
# End tblastn of mouse proteins


####################################################################################
# RE-BUILD KNOWN GENES TABLES, 3ND TRIAL WITH CORRECTED kgCheck and kgGetCds (DONE 6/5/05 Fan)

# Start from the step that gene-check is run and kgCandidate0.gp is produced.

   cd
   cd /cluster/store10/kg/kgHg17F

   mkdir try3
   cd try3
   hgsql kgHg17FTempTry3 -e 'drop table kgCandidate0'

   hgsql kgHg17FTempTry3 -e 'drop table kgCandidate0'
   hgsql kgHg17FTempTry3 < ~/src/hg/lib/kgCandidate0.sql 
   hgsql kgHg17FTempTry3 -e  'load data local infile "../kgCandidate0.gp" into table kgCandidate0'

   hgsql kgHg17FTempTry3 -e 'drop table geneCheck'
   hgsql kgHg17FTempTry3 < ~/src/hg/lib/geneCheck.sql
   hgsql kgHg17FTempTry3 -e  'load data local infile "../kgCandidate0.check" into table geneCheck ignore 2 lines'

# Run kgCheck to get all KG candidates that pass the KG gene check criteria

   kgCheck kgHg17FTempTry3 hg17 kgCandidate0 geneCheck kgCandidate.tab
   hgsql kgHg17FTempTry3 -e  'drop table kgCandidate'
   hgsql kgHg17FTempTry3 < ~/src/hg/lib/kgCandidate.sql
   hgsql kgHg17FTempTry3 -e  'load data local infile "kgCandidate.tab" into table kgCandidate'
   hgsql kgHg17FTempTry3 -e 'create index alignID on kgCandidate(alignID)'

# Construct the kgCandidateX table that has alignID in the name field. 
   cut -f 2-10 kgCandidate.tab >j2.tmp
   cut -f 11 kgCandidate.tab >j1.tmp
   paste j1.tmp j2.tmp >kgCandidateX.tab

   hgsql kgHg17FTempTry3 -e  'drop table kgCandidateX'
   hgsql kgHg17FTempTry3 < ~/src/hg/lib/kgCandidateX.sql
   hgsql kgHg17FTempTry3 -e  'load data local infile "kgCandidateX.tab" into table kgCandidateX'

# Score protein/mRna and protein/RefSeq alignments

#   kgResultBestMrna2 050415 kgHg17FTempTry3 hg17|sort -u >protMrnaBlatScore.tab
#   kgResultBestRef2  050415 kgHg17FTempTry3 hg17|sort -u >protRefScore.tab

# Combine scoring results and load them into temp DB.
   cat protMrnaBlatScore.tab protRefScore.tab >protMrnaScore.tab
   hgsql kgHg17FTempTry3 -e 'drop table protMrnaScore'
   hgsql kgHg17FTempTry3 < ~/src/hg/lib/protMrnaScore.sql
   hgsql kgHg17FTempTry3 -e 'load data local infile "../protMrnaScore.tab" into table protMrnaScore'
   hgsql kgHg17FTempTry3 -e 'create index mrnaAcc on protMrnaScore(mrnaAcc)'


# Run kgGetCds to get CDS structure of each gene

   kgGetCds kgHg17FTempTry3 kgCandidateX jY.tmp1
   cat jY.tmp1 |sort -u >kgCandidateY.tab
   rm jY.tmp1
   hgsql kgHg17FTempTry3 -e  'drop table kgCandidateY'
   hgsql kgHg17FTempTry3 < ~/src/hg/lib/kgCandidateY.sql
   hgsql kgHg17FTempTry3 -e  'load data local infile "kgCandidateY.tab" into table kgCandidateY'

# Run kgPickPrep to replace long cds structure string with cdsId.
   kgPickPrep kgHg17FTempTry3 kgCandidateZ.tab
   hgsql kgHg17FTempTry3 -e  'drop table kgCandidateZ'
   hgsql kgHg17FTempTry3 < ~/src/hg/lib/kgCandidateZ.sql
   hgsql kgHg17FTempTry3 -e  'load data local infile "kgCandidateZ.tab" into table kgCandidateZ'
   hgsql kgHg17FTempTry3 -e 'create index cdsId on kgCandidateZ(cdsId)'

# Run kgPick to pick the representative a mrna/protein pair for each unique CDS structure.

   kgPick kgHg17FTempTry3 hg17 proteins050415 kg3Try3.tmp dupSpMrna.tmp

   cat kg3Try3.tmp | grep NM_ > jNM
   cat kg3Try3.tmp | grep -v NM_ >jnoNM
   cut -f 1 jnoNM | sed -e "s/_/_\n/" |grep -v _ >jnoNM1
   cut -f 2-12  jnoNM >jnoNM2
   paste jnoNM1 jnoNM2 > kg3Try3B.tmp
   cat jNM >> kg3Try3B.tmp

   sort -u dupSpMrna.tmp >dupSpMrna.tab

   hgsql hg17 -e 'drop table dupSpMrna'
   hgsql hg17 < ~/src/hg/lib/dupSpMrna.sql
   hgsql hg17 -e  'load data local infile "dupSpMrna.tab" into table dupSpMrna'


# Add entries in the put back list
# Obtain from Mark the put back list, kgPutBack.lis, for human RefSeq.

   hgsql kgHg17FTempTry3 -e 'drop table kgPutBack'
   hgsql kgHg17FTempTry3 < ~/src/hg/lib/kgPutBack.sql
   hgsql kgHg17FTempTry3 -e  'load data local infile "kgPutBack.lis" into table kgPutBack'

   kgPutBack kgHg17FTempTry3 hg17 proteins050415 kgPutBack kgPutBack.gp

# Sort KG genes to make the kg3Try3.gp table file.

   cat kg3Try3B.tmp kgPutBack.gp >kg3Try3C.tmp
   ~/kent/src/hg/protein/sortKg.pl kg3Try3C.tmp >kg3Try3.gp

# Manually edit to correct one line problem of O75438_BC009691

   hgsql kgHg17FTempTry3 -e  'drop table knownGene'
   hgsql kgHg17FTempTry3 < ~/src/hg/lib/knownGene.sql
   hgsql kgHg17FTempTry3 -e  'load data local infile "kg3Try3.gp" into table knownGene'

   hgsql hg17 -e  'drop table kg3Try3'
   hgsql hg17 < ~/src/hg/lib/kg3Try3.sql
   hgsql hg17 -e  'load data local infile "kg3Try3.gp" into table kg3Try3'

# Perform analysis before renaming the kg3Try3 table to knownGene.

# Load data into hg17 knownGene table.
   hgsql hg17 -e  'drop table knownGene'
   hgsql hg17 < ~/src/hg/lib/knownGene.sql
   hgsql hg17 -e  'load data local infile "kg3Try3.gp" into table knownGene'

# Build knownGeneMrna and knownGenePep tables.
   hgsql kgHg17FTempTry3 -e  'drop table mrnaSeq'
   hgsql kgHg17FTempTry3 < ~/src/hg/lib/mrnaSeq.sql
   hgsql kgHg17FTempTry3 -e  'load data local infile "../mrnaSeq.tab" into table mrnaSeq'

   kgPepMrna kgHg17FTempTry3 hg17 050415

   hgsql hg17 -e  'drop table knownGeneMrna'
   hgsql hg17 < ~/src/hg/lib/knownGeneMrna.sql
   hgsql hg17 -e  'load data local infile "knownGeneMrna.tab" into table knownGeneMrna'
   hgsql hg17 -e  'drop table knownGenePep'
   hgsql hg17 < ~/src/hg/lib/knownGenePep.sql
   hgsql hg17 -e  'load data local infile "knownGenePep.tab" into table knownGenePep'

# Build spMrna table

   hgsql hg17 -N -e 'select proteinID, name from knownGene' |sort -u >kgSpMrna.tab

   hgsql hg17 -e  'drop table spMrna'
   hgsql hg17 <~/src/hg/lib/spMrna.sql
   hgsql hg17 -e 'load data local infile "kgSpMrna.tab" into table spMrna'

# Build kgXref table

   kgXref2 kgHg17FTempTry3 050415 hg17

   hgsql hg17 -e  'drop table kgXref'
   hgsql hg17 < ~/src/hg/lib/kgXref.sql
   hgsql hg17 -e  'load data local infile "kgXref.tab" into table kgXref'
 
# MAKE FOLDUTR TABLES 
# First set up directory structure and extract UTR sequence on hgwdev
    ssh hgwdev
    cd /cluster/data/hg17/bed
    mkdir rnaStruct.2005-06-05
    rm rnaStruct
    ln -s rnaStruct.2005-06-05 rnaStruct
    cd rnaStruct
    mkdir -p utr3/split utr5/split utr3/fold utr5/fold
    utrFa hg17 knownGene utr3 utr3/utr.fa
    utrFa hg17 knownGene utr5 utr5/utr.fa

# Split up files and make files that define job.
    ssh kk
    cd /cluster/data/hg17/bed/rnaStruct
    faSplit sequence utr3/utr.fa 50000 utr3/split/s
    faSplit sequence utr5/utr.fa 50000 utr5/split/s
    ls -1 utr3/split > utr3/in.lst
    ls -1 utr5/split > utr5/in.lst
    cd utr3
    cat > gsub <<end
#LOOP
rnaFoldBig split/\$(path1) fold
#ENDLOOP
end
    cp gsub ../utr5

# Do cluster run for 3' UTRs
    gensub2 in.lst single gsub spec
    para create spec
    para try
    para push
# Completed: 35774 of 35774 jobs
# CPU time in finished jobs:    1174534s   19575.57m   326.26h   13.59d  0.037 y
# IO & Wait Time:                 98071s    1634.51m    27.24h    1.14d  0.003 y
# Average job time:                  36s       0.59m     0.01h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:            5409s      90.15m     1.50h    0.06d
# Submission to last job:          6712s     111.87m     1.86h    0.08d

# Do cluster run for 5' UTRs 
    cd ../utr5
    gensub2 in.lst single gsub spec
    para create spec
    para try
    para push
# Completed: 33765 of 33765 jobs
# CPU time in finished jobs:     341000s    5683.33m    94.72h    3.95d  0.011 y
# IO & Wait Time:                106605s    1776.75m    29.61h    1.23d  0.003 y
# Average job time:                  13s       0.22m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:           30479s     507.98m     8.47h    0.35d
# Submission to last job:         30622s     510.37m     8.51h    0.35d 

# Load database
    ssh hgwdev
    cd /cluster/data/hg17/bed/rnaStruct/utr5
    hgLoadRnaFold hg17 foldUtr5 fold
    cd ../utr3
    hgLoadRnaFold hg17 foldUtr3 fold

# Clean up
    rm -r split fold err batch.bak
    cd ../utr5
    rm -r split fold err batch.bak

# Build kgProtMap table

# move all files under hg17Kg to old and copy try3/kgXref.tab up.

# Note: it is important that tight_mrna.psl is here!
    cp old/tight_mrna.psl . -p

    ~/src/hg/protein/kgProtMap2.sh kgHg17F hg17 050415

# Update and clean up kgResultBestMrna2.c and then check it in.

# Build alias tables

#	kgAliasM reads from proteins050415.hugo.symbol, proteins050415.hugo.aliases
#	proteins050415.hugo.withdraws, hg17.kgXref.kgID
#	to create kgAliasM.tab and geneAlias.tab
#	by picking out those kgID items from kgXref where
#	kgXref.geneSymbol == hugo.symbol

   kgAliasM hg17 proteins050415

#	kgAliasKgXref reads from hg17.knownGene.proteinID,
#	hg17.knownGene.name, hg17.kgXref.geneSymbol
#	to create kgAliasKgXref.tab

   kgAliasKgXref hg17


#	kgAliasRefseq reads from hg17.knownGene.name,
#	hg17.knownGene.proteinID, hg17.kgXref.refseq
#	to create kgAliasRefseq.tab

   kgAliasRefseq hg17

   hgsql sp050415 -N -e 'select name,gene.val from hg17.knownGene,displayId,gene where displayId.val=proteinID and displayId.acc=gene.acc' \
   | sort -u  > kgAliasP.tab

   hgsql hg17 -N -e 'select name, name from knownGene' >kgAliasDup.tab
   hgsql hg17 -N -e 'select mrnaID, dupMrnaID from dupSpMrna' >>kgAliasDup.tab
   
   cat kgAliasM.tab kgAliasRefseq.tab kgAliasKgXref.tab kgAliasP.tab kgAliasDup.tab| \
   sort |uniq > kgAlias.tab

   hgsql -e "drop table kgAlias;" hg17 
   hgsql hg17 < ~/kent/src/hg/lib/kgAlias.sql
   hgsql hg17 -e 'LOAD DATA local INFILE "kgAlias.tab" into table kgAlias' 

#	kgProtAlias reads from hg17.knownGene.name,
#	hg17.knownGene.proteinID, hg17.knownGene.alignID,
#	proteins050415.spXref3.accession, proteins050415.spSecondaryID, proteins050415.pdbSP.pdb
#	to create kgProtAlias.tab
#

   kgProtAlias hg17 050415

   hgsql hg17 -N -e \
   'select kgID, spDisplayID, protAcc from kgXref where protAcc != ""'\
   | sort -u >kgProtAliasNCBI.tab

# include variant splice protein IDs
   
   hgsql hg17 -N -e \
   'select name, proteinID, parAcc from knownGene,sp050415.varAcc where varAcc=proteinID'\
   |sort -u >kgProtAliasDup.tab

# include duplicate protein IDs from dupSpMrna table
   hgsql hg17 -N -e \
   'select name, knownGene.proteinID, dupProteinID from knownGene, dupSpMrna where name=mrnaID'\
   |sort -u >>kgProtAliasDup.tab

# catch parent acc from dupProteinID too

   hgsql hg17 -N -e\
   'select name, knownGene.proteinID, parAcc from knownGene,dupSpMrna,sp050415.varAcc where name=mrnaID and dupProteinID=varAcc.varAcc'\
   |sort -u >>kgProtAliasDup.tab
    cat kgProtAliasNCBI.tab kgProtAlias.tab kgProtAliasDup.tab | sort -u > kgProtAliasAll.tab

    echo "`date` creating table kgProtAlias"
    hgsql hg17 -e "drop table kgProtAlias;"
    hgsql hg17 <~/src/hg/lib/kgProtAlias.sql; 
    hgsql hg17 -e 'LOAD DATA local INFILE "kgProtAliasAll.tab" into table kgProtAlias;'  

# Build kgSpAlias table

    hgsql hg17 -e \
    'select kgXref.kgID, spID, alias from kgXref, kgAlias where kgXref.kgID=kgAlias.kgID' >j.tmp
    hgsql hg17 -e \
    'select kgXref.kgID, spID, alias from kgXref, kgProtAlias where kgXref.kgID=kgProtAlias.kgID'\
    >>j.tmp
    cat j.tmp|sort -u |grep -v 'kgID' >hg17.kgSpAlias.tab
    rm j.tmp

    hgsql hg17 -e 'drop table kgSpAlias';
    hgsql hg17 < ~/src/hg/lib/kgSpAlias.sql
    hgsql hg17 -e 'load data local infile "hg17.kgSpAlias.tab" into table kgSpAlias'
    

# Build KEGG pathway tables

   ssh hgwdev
   cd /cluster/store10/kg/kgHg17F
   md kegg
   cd kegg

   ~/src/hg/protein/KGpath.sh kgHg17F hg17 050415

   hgsql hg17 -e "drop table keggMapDesc"
   hgsql hg17 -e "drop table keggPathway"
   hgsql hg17 <~/src/hg/lib/keggMapDesc.sql
   hgsql hg17 <~/src/hg/lib/keggPathway.sql
   hgsql hg17 -e 'load data local infile "keggMapDesc.tab" into table keggMapDesc'
   hgsql hg17 -e 'load data local infile "keggPathway.tab" into table keggPathway'

# Build CGAP pathway tables

   cd ..
   ~/src/hg/protein/KGcgap.sh kgHg17F hg17 050415
   hgsql hg17 -e "drop table cgapAlias"
   hgsql hg17 -e "drop table cgapBiocDesc"
   hgsql hg17 -e "drop table cgapBiocPathway"
   hgsql hg17 <~/src/hg/lib/cgapAlias.sql
   hgsql hg17 <~/src/hg/lib/cgapBiocDesc.sql
   hgsql hg17 <~/src/hg/lib/cgapBiocPathway.sql
   hgsql hg17 -e 'load data local infile "cgapAlias.tab" into table cgapAlias'
   hgsql hg17 -e 'load data local infile "cgapBIOCARTAdescSorted.tab" into table cgapBiocDesc'
   hgsql hg17 -e 'load data local infile "cgapBIOCARTA.tab" into table cgapBiocPathway'

# Build BioCyc pathway tables

# Download BioCyc DB, create and load bioCyc DB
# See makeBioCycDB.doc for details.
   
   hgsql hg17 -e "drop table bioCycMapDesc"
   hgsql hg17 <~/src/hg/lib/bioCycMapDesc.sql
   hgsql hg17 -e 'load data local infile "bioCycMapDesc.tab" into table bioCycMapDesc'

   kgBioCyc |sort -u > bioCycPathway.tab

   hgsql hg17 -e "drop table bioCycPathway"
   hgsql hg17 <~/src/hg/lib/bioCycPathway.sql
   hgsql hg17 -e 'load data local infile "bioCycPathway.tab" into table bioCycPathway'

# CCDS <-> knownGene mapping need to be updated (Fan redone 2005-06-05)
# this should be part of the known gene build
    /cluster/data/genbank/bin/i386/mkCcdsGeneMap -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap


### HG17 PROTEOME BROWSER TABLES RE-BUILD ####  (DONE - 2005-06-05 - Fan)
# These are instructions for rebuilding tables 
# needed for the Proteome Browser.  
# DON'T START THESE UNTIL TABLES FOR KNOWN GENES AND kgProtMap table
# ARE REBUILT.  
# This update is based on proteins DBs dated 050415.

# Create the working directory

    ssh hgwdev
    mkdir /cluster/store10/kg/kgHg17F/pb-2005-06-05
    cd /cluster/data/hg17/bed
    rm pb
    ln -s /cluster/store10/kg/kgHg17F/pb-2005-06-05 pb
    cd pb

# Move the existing PB tables by:

hgsql hg17
create database hg17Sav4;

alter table hg17.pepCCntDist rename as hg17Sav4.pepCCntDist;
alter table hg17.pepExonCntDist rename as hg17Sav4.pepExonCntDist;
alter table hg17.pepHydroDist rename as hg17Sav4.pepHydroDist;
alter table hg17.pepIPCntDist rename as hg17Sav4.pepIPCntDist;
alter table hg17.pepMolWtDist rename as hg17Sav4.pepMolWtDist;
alter table hg17.pepMwAa rename as hg17Sav4.pepMwAa;
alter table hg17.pepPi rename as hg17Sav4.pepPi;
alter table hg17.pepPiDist rename as hg17Sav4.pepPiDist;
alter table hg17.pepResDist rename as hg17Sav4.pepResDist;

alter table hg17.pbAnomLimit rename as hg17Sav4.pbAnomLimit;
alter table hg17.pbResAvgStd rename as hg17Sav4.pbResAvgStd;
alter table hg17.pbStamp rename as hg17Sav4.pbStamp;

quit

# Define pep* tables in hg17 DB

	cat ~/kent/src/hg/lib/pep*.sql > pepAll.sql

#  First edit out pepPred table definition, then

	hgsql hg17 < pepAll.sql

# Build the pepMwAa table

  hgsql proteins050415 -N -e \
"select info.acc, molWeight, aaSize from sp050415.info, sp050415.accToTaxon where accToTaxon.taxon=9606 and accToTaxon.acc = info.acc" > pepMwAa.tab

hgsql hg17 -e 'load data local infile "pepMwAa.tab" into table pepMwAa'

o Build the pepPi table

    hgsql proteins050415 -e \
    "select info.acc from sp050415.info, sp050415.accToTaxon where accToTaxon.taxon=9606 and accToTaxon.acc = info.acc" > protAcc.lis

    hgsql hg17 -N -e 'select proteinID from knownGene where proteinID like "%-%"' | sort -u >> protAcc.lis

    pbCalPi protAcc.lis sp050415 pepPi.tab
    hgsql hg17 -e 'delete from pepPi'
    hgsql hg17 -e 'load data local infile "pepPi.tab" into table hg17.pepPi'

# Calculate and load pep distributions

    pbCalDist sp050415 proteins050415 9606 hg17 >pbCalDist.out
    wc  pbCalDist.out

    hgsql hg17
    load data local infile "pepExonCntDist.tab" into table hg17.pepExonCntDist;
    load data local infile "pepCCntDist.tab" into table hg17.pepCCntDist;
    load data local infile "pepHydroDist.tab" into table hg17.pepHydroDist;
    load data local infile "pepMolWtDist.tab" into table hg17.pepMolWtDist;
    load data local infile "pepResDist.tab" into table hg17.pepResDist;
    load data local infile "pepIPCntDist.tab" into table hg17.pepIPCntDist;
    load data local infile "pepPiDist.tab" into table hg17.pepPiDist;
    quit

# Calculate frequency distributions

    pbCalResStd sp050415 9606 hg17

# Create pbAnomLimit and pbResAvgStd tables

   hgsql hg17 -e "drop table pbAnomLimit"
   hgsql hg17 -e "drop table pbResAvgStd"
   hgsql hg17 < ~/src/hg/lib/pbAnomLimit.sql
   hgsql hg17 < ~/src/hg/lib/pbResAvgStd.sql

   hgsql hg17 -e 'load data local infile "pbResAvgStd.tab" into table hg17.pbResAvgStd;'
   hgsql hg17 -e 'load data local infile "pbAnomLimit.tab" into table hg17.pbAnomLimit;'

# Create pbStamp table for PB
  hgsql hg17 -e "drop table pbStamp"
  hgsql hg17 < ~/src/hg/lib/pbStamp.sql
  hgsql hg17Sav4 -N -e 'select * from pbStamp' > pbStamp.tab
  hgsql hg17 -e 'load data local infile "pbStamp.tab" into table hg17.pbStamp'

# Adjust drawing parameters for Proteome Browser stamps

  Now invoke Proteome Browser and adjust various drawing parameters
  (mostly the ymax of each stamp) if necessary, by updating the 
  pbStamp.tab file and then delete and reload the pbStamp table. 

# Perform preliminary review of Proteome Browser for hg17, then
  notify QA for formal review.


# RE-BUILD GENE SORTER TABLES (AKA: FAMILY BROWSER) (DONE - 2005-06-04 - Fan)
#	This should be done after KG tables are complete from known genes build
#	process.
#
# Cluster together various alt-splicing isoforms.
#	Creates the knownIsoforms and knownCanonical tables
ssh hgwdev
mkdir /cluster/data/hg17/bed/geneSorter.2005-06-04
# remove old symbolic link
rm /cluster/data/hg17/bed/geneSorter
ln -s /cluster/data/hg17/bed/geneSorter.2005-06-04 /cluster/data/hg17/bed/geneSorter
cd /cluster/data/hg17/bed/geneSorter
hgClusterGenes hg17 knownGene knownIsoforms knownCanonical

# Extract peptides from knownGenes into fasta file
# and create a blast database out of them.
mkdir /cluster/data/hg17/bed/geneSorter/blastp
cd /cluster/data/hg17/bed/geneSorter/blastp
pepPredToFa hg17 knownGenePep known.faa
#	You may need to build this binary in src/hg/near/pepPredToFa
/scratch/blast/formatdb -i known.faa -t known -n known
#	This command is in /projects/compbio/bin/$MACH/formatdb

# Copy over database to bluearc
rm -fr /cluster/bluearc/hg17/blastp
mkdir -p /cluster/bluearc/hg17/blastp
cp -p /cluster/data/hg17/bed/geneSorter/blastp/known.* /cluster/bluearc/hg17/blastp

# Split up fasta file into bite sized chunks for cluster
cd /cluster/data/hg17/bed/geneSorter/blastp
mkdir split
faSplit sequence known.faa 8000 split/kg

# Make parasol run directory
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/self
cd /cluster/data/hg17/bed/geneSorter/blastp/self
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
-p blastp -d /cluster/bluearc/hg17/blastp/known -i $1 -o $2 \
-e 0.01 -m 8 -b 1000
'_EOF_'
    # << keep emacs happy
chmod +x blastSome

# Make gensub2 file
cat  << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#	'ls ../../split/*.fa' is too much, hence the echo
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para push
para check
# Completed: 7735 of 7735 jobs
# CPU time in finished jobs:     142764s    2379.39m    39.66h    1.65d  0.005 y
# IO & Wait Time:                 67623s    1127.06m    18.78h    0.78d  0.002 y
# Average job time:                  27s       0.45m     0.01h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:             144s       2.40m     0.04h    0.00d
# Submission to last job:           392s       6.53m     0.11h    0.00d

# Load into database.  This takes about 30 minutes
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/self/run/out
time hgLoadBlastTab hg17 knownBlastTab *.tab
# Scanning through 7735 files
# Loading database with 9757382 rows
# 255.200u 50.520s 25:19.66 20.1% 0+0k 0+0io 247pf+0w

cd /cluster/data/hg17/bed/geneSorter
# Create table that maps between known genes and RefSeq
hgMapToGene hg17 refGene knownGene knownToRefSeq
#	may need to build this command in src/hg/near/hgMapToGene
#	hgsql -e "select count(*) from knownToRefSeq;" hg17
#	row count changed 34667

# Create table that maps between known genes and LocusLink
hgsql --skip-column-names -e "select mrnaAcc,locusLinkId from refLink" hg17 > refToLl.txt
hgMapToGene hg17 refGene knownGene knownToLocusLink -lookup=refToLl.txt
#	hgsql -e "select count(*) from knownToLocusLink;" hg17
#	row count changed to 34773

# Create table that maps between known genes and Pfam domains
hgMapViaSwissProt hg17 knownGene name proteinID Pfam knownToPfam
#	hgsql -e "select count(*) from knownToPfam;" hg17
#	row count changed to 29171 

# Create table to map between known genes and GNF Atlas2
# expression data.
    hgMapToGene hg17 gnfAtlas2 knownGene knownToGnfAtlas2 '-type=bed 12'
#	hgsql -e "select count(*) from knownToGnfAtlas2;" hg17
#	row count changed to 32458

# Create expression distance table - takes about an hour
    hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
    	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
	-lookup=knownToGnfAtlas2 &
# Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
# Got 32458 unique elements in hgFixed.gnfHumanAtlas2MedianRatio
#	hgsql -e "select count(*) from gnfAtlas2Distance;" hg17
#	row count changed to 32381000

# Create a table that maps between known genes and 
# the nice affy expression data.
hgMapToGene "-type=bed 12" hg17 affyUclaNorm knownGene knownToU133
#	hgsql -e "select count(*) from knownToU133;" hg17
#	row count changed to 32965

# Create expression distance table.  This will take about 2.5 hours
cd /tmp
cp -p ~/kent/src/hg/near/hgExpDistance/affyUcla.weight .
time hgExpDistance hg17 affyUclaNorm affyUclaExp knownExpDistance \
	-weights=affyUcla.weight -lookup=knownToU133 &
# Have 43039 elements in affyUclaNorm
# 211 genes, 42 weights, 26.500000 total wieght
# Got 32965 unique elements in affyUclaNorm

# Create table that maps between known genes and 
# the GNF data.
cd /tmp
hgMapToGene hg17 affyU95 knownGene knownToU95
#	row count changed to 17555
#	hgFixed.gnfHumanU95Exps argument is unused, no need to exist
hgExpDistance hg17 hgFixed.gnfHumanU95MedianRatio \
	hgFixed.gnfHumanU95Exps gnfU95Distance  -lookup=knownToU95 &
# Have 11545 elements in hgFixed.gnfHumanU95MedianRatio
# Got 16501 unique elements in hgFixed.gnfHumanU95MedianRatio
#	row count changed to 16450000  

# Create known gene mapping table and expression distance tables
# for GNF Atlas 2.  (The hgExpDistance takes only 10 minutes.)

hgMapToGene hg17 affyGnf1h knownGene knownToGnf1h
hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
	hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
	-lookup=knownToGnf1h &
# Have 44696 elements in hgFixed.gnfHumanAtlas2MedianRatio
# Got 8827 unique elements in hgFixed.gnfHumanAtlas2MedianRatio

cd /cluster/data/hg17/bed/geneSorter
hgMapToGene hg17 affyU133Plus2 knownGene knownToU133Plus2
#	row count changed to 35139  

#### UPDATE GO DATABASE (THIS PART WAS DONE 5/21/05 Fan)

# Download the terms and make the database.
ssh hgwdev
mkdir /cluster/store1/geneOntology/20050521
cd /cluster/store1/geneOntology/20050521

wget --timestamping http://www.godatabase.org/dev/database/archive/latest/go_200504-assocdb-data.gz

hgsql mysql <<end
create database go050521;
end
zcat go_*data.gz | sed -e 's/ENGINE=MyISAM DEFAULT CHARSET=latin1/TYPE=MyISAM/g' >j.tmp
hgsql go050521 <j.tmp
rm j.tmp

wget --timestamping ftp://ftp.geneontology.org/pub/go/gene-associations/gene_association.goa_uniprot.gz

zcat gene_association.goa_uniprot.gz | hgGoAssociation go050521 goaPart stdin 
# Passed 5589891 of 6584507 of 6584507, 84.89%

# Ask sys-admin to switch the database pointer go to point to go050521.

cd /cluster/data/hg17/bed/geneSorter

# Rebuilt Ensembl Gene tables.  See documentation (5/23/05 Fan) above.

# Create knownToEnsembl column
hgMapToGene hg17 ensGene knownGene knownToEnsembl
# hgsql hg17 -e "select count(*) from knownToEnsembl" 
#	table row count 35521

# Make knownToCdsSnp table 
  ssh hgwdev
  nice hgMapToGene hg17 snp knownGene knownToCdsSnp -all -cds
# hgsql hg17 -e "select count(*) from knownToCdsSnp" 
# row count 94633
# approx. 5 minutes running time

# C.ELEGANS BLASTP FOR GENE SORTER 
    # Make C. elegans ortholog column using blastp on wormpep.
    # First make C. elegans protein database and copy it to iscratch/i
    # if it doesn't exist already:

# The following section is done during mm6 build already.
#    ssh eieio
#    mkdir /cluster/data/ce2/bed/blastp
#    cd /cluster/data/ce2/bed/blastp
#    # Point a web browser at ftp://ftp.sanger.ac.uk/pub/databases/wormpep/
#    # to find out the latest version.  Then use that in place of 142 below.
#    wget -O wormPep142.faa ftp://ftp.sanger.ac.uk/pub/databases/wormpep/wormpep142/wormpep142
#    formatdb -i wormPep142.faa -t wormPep142 -n wormPep142
#    ssh kkr1u00
#    if (-e /iscratch/i/ce2/blastp) then
#      rm -r /iscratch/i/ce2/blastp
#    endif
#    mkdir -p /iscratch/i/ce2/blastp
#    cp /cluster/data/ce2/bed/blastp/wormPep142.p?? /iscratch/i/ce2/blastp
#    iSync

    # Make parasol run directory 
    ssh kk
    mkdir -p /cluster/data/hg17/bed/blastp/ce2/run/out
    cd /cluster/data/hg17/bed/blastp/ce2/run
    # Make blast script
    cat > blastSome <<end
#!/bin/csh
setenv BLASTMAT /iscratch/i/blast/data
/iscratch/i/blast/blastall -p blastp -d /iscratch/i/ce2/blastp/wormPep142 -i \$1 -o \$2 -e 0.01 -m 8 -b 1
end
    chmod a+x blastSome
    # Make gensub2 file
    cat > gsub <<end
#LOOP
blastSome {check in line+ \$(path1)} {check out line out/\$(root1).tab}
#ENDLOOP
end
    # Create parasol batch
    ls -1S /cluster/data/hg17/bed/geneSorter/blastp/split \
    |sed -e 's=kg=../../../geneSorter/blastp/split/kg=g' >split.lst
    gensub2 split.lst single gsub spec
    para create spec
    para try, check, push, check, ...  

# Initial run has 13 jobs crashed.
# crashed: 13
# ranOk: 7722

# para problems show the following typical message:

# total jobs in batch: 7735
# job: blastSome ../../../geneSorter/blastp/split/kg5911.fa out/kg5911.tab
# id: 209522384
# failure type: crash
# host: kkr2u28.kilokluster.ucsc.edu
# start time: Sat Jun  4 11:45:51 2005
# return: 0
# stderr:
# [blastall] FATAL ERROR: blast: Unable to open input file ../../../geneSorter/blastp/split/kg5911.fa

# para push again and these 13 ran fine.
# Completed: 7735 of 7735 jobs
# CPU time in finished jobs:      60319s    1005.32m    16.76h    0.70d  0.002 y
# IO & Wait Time:                 31239s     520.65m     8.68h    0.36d  0.001 y
# Average job time:                  12s       0.20m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              72s       1.20m     0.02h    0.00d
# Submission to last job:           199s       3.32m     0.06h    0.00d

# Load into database.  
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastp/ce2/run/out
    hgLoadBlastTab hg17 ceBlastTab -maxPer=1 *.tab
# Scanning through 7735 files
# Loading database with 25574 rows

# Make mouse ortholog column using blastp on mouse known genes.
# First make mouse protein database and copy it to /cluster/panasas
# if it doesn't exist already
#	This already exists.  See makeMm6.doc for procedure

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/mm6
cd /cluster/data/hg17/bed/geneSorter/blastp/mm6
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
-p blastp -d /cluster/panasas/home/store/mm6/blastp/known \
-i $1 -o $2 -e 0.001 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#	this echo trick is used because otherwise the command line is
#	too long and you can not do a simple ls
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7735 of 7735 jobs
# CPU time in finished jobs:      85769s    1429.49m    23.82h    0.99d  0.003 y
# IO & Wait Time:                 20587s     343.11m     5.72h    0.24d  0.001 y
# Average job time:                  14s       0.23m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              78s       1.30m     0.02h    0.00d
# Submission to last job:           206s       3.43m     0.06h    0.00d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/mm6/run/out
hgLoadBlastTab hg17 mmBlastTab -maxPer=1 *.tab
# Scanning through 7735 files
# Loading database with 32951 rows

# Make rat ortholog column using blastp on rat known genes.
# First make rat protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This already exists.  See makeRn3.doc for procedure.
#	Files were put in this directory: /cluster/bluearc/rn3/blastp/

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/rn3
cd /cluster/data/hg17/bed/geneSorter/blastp/rn3
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
-p blastp -d /cluster/bluearc/rn3/blastp/known \
-i $1 -o $2 -e 0.001 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#	this echo trick is used because otherwise the command line is
#	too long and you can not do a simple ls
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc 
# Completed: 7735 of 7735 jobs
# CPU time in finished jobs:      27804s     463.40m     7.72h    0.32d  0.001 y
# IO & Wait Time:                 30334s     505.56m     8.43h    0.35d  0.001 y
# Average job time:                   8s       0.13m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              26s       0.43m     0.01h    0.00d
# Submission to last job:           119s       1.98m     0.03h    0.00d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/rn3/run/out
hgLoadBlastTab hg17 rnBlastTab -maxPer=1 *.tab
# Scanning through 7735 files
# Loading database with 24030 rows

# ZEBRAFISH BLASTP FOR GENE SORTER 
    # Make Danio rerio (zebrafish) ortholog column using blastp on Ensembl.
    # First make protein database and copy it to iscratch/i
    # if it doesn't exist already:
    ssh kkstore
    mkdir /cluster/data/danRer2/bed/blastp
    cd /cluster/data/danRer2/bed/blastp
    wget ftp://ftp.ensembl.org/pub/current_zebrafish/data/fasta/pep/Danio_rerio.ZFISH4.may.pep.fa.gz 
    zcat Dan*.pep.fa.gz > ensembl.faa
    /scratch/blast/formatdb -i ensembl.faa -t ensembl -n ensembl
    ssh kkr1u00
    if (-e /iscratch/i/danRer2/blastp) then
      rm -r /iscratch/i/danRer2/blastp
    endif
    mkdir -p /iscratch/i/danRer2/blastp
    cp /cluster/data/danRer2/bed/blastp/ensembl.p?? /iscratch/i/danRer2/blastp
    iSync

    # Make parasol run directory 
    ssh kk
    mkdir -p /cluster/data/hg17/bed/blastp/danRer2/run/out
    cd /cluster/data/hg17/bed/blastp/danRer2/run
    # Make blast script
    cat > blastSome <<end
#!/bin/csh
setenv BLASTMAT /iscratch/i/blast/data
/iscratch/i/blast/blastall -p blastp -d /iscratch/i/danRer2/blastp/ensembl -i \$1 -o \$2 -e 0.005 -m 8 -b 1
end
    chmod a+x blastSome
    # Make gensub2 file
    cat > gsub <<end
#LOOP
blastSome {check in line+ \$(path1)} {check out line out/\$(root1).tab}
#ENDLOOP
end
    # Create parasol batch
    ls -1S /cluster/data/hg17/bed/geneSorter/blastp/split \
    |sed -e 's=kg=../../../geneSorter/blastp/split/kg=g' > split.lst
    gensub2 split.lst single gsub spec
    para create spec
    para try, check, push, check, ...
# Completed: 7735 of 7735 jobs
# CPU time in finished jobs:     111467s    1857.78m    30.96h    1.29d  0.004 y
# IO & Wait Time:                 21159s     352.65m     5.88h    0.24d  0.001 y
# Average job time:                  17s       0.29m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              95s       1.58m     0.03h    0.00d
# Submission to last job:           223s       3.72m     0.06h    0.00d

    # Load into database.  
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastp/danRer2/run/out
    hgLoadBlastTab hg17 drBlastTab -maxPer=1 *.tab
# Scanning through 7735 files
# Loading database with 30651 rows

# Make Saccharomyces cerevisiae (yeast) ortholog column using blastp on RefSeq.
# First make protein database and copy it to cluster/bluearc
# if it doesn't exist already
#	This is already done, see makeMm3.doc for procedure
#	the directory: /cluster/bluearc/sc1/blastp should have data

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/sc1
cd /cluster/data/hg17/bed/geneSorter/blastp/sc1
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
-p blastp -d /cluster/bluearc/sc1/blastp/sgd \
-i $1 -o $2 -e 0.01 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7735 of 7735 jobs
# CPU time in finished jobs:      18194s     303.23m     5.05h    0.21d  0.001 y
# IO & Wait Time:                 24452s     407.53m     6.79h    0.28d  0.001 y
# Average job time:                   6s       0.09m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              16s       0.27m     0.00h    0.00d
# Submission to last job:           120s       2.00m     0.03h    0.00d
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/sc1/run/out
hgLoadBlastTab hg17 scBlastTab -maxPer=1 *.tab
# Scanning through 7735 files
# Loading database with 16395 rows 

# Make Drosophila melanagaster ortholog column using blastp on FlyBase.
# First make SwissProt protein database and copy it to cluster/bluearc
# The following section was already done.
# cd /cluster/data/dm1/bed
# mkdir blastp
# cd blastp
#wget ftp://ftp.fruitfly.org/pub/download/dmel_RELEASE3-1/FASTA/whole_genome_translation_dmel_RELEASE3-1.FASTA.gz
# zcat whole_ge*.gz | faFlyBaseToUcsc stdin flyBase.faa
# formatdb -i flyBase.faa -t flyBase -n flyBase
# if (-e /cluster/bluearc/dm1/blastp) then
#    rm -r /cluster/bluearc/dm1/blastp
# endif
# mkdir -p /cluster/bluearc/dm1/blastp
# cp /cluster/data/dm1/bed/blastp/flyBase.p?? /cluster/bluearc/dm1/blastp

# Make parasol run directory 
ssh kk
mkdir /cluster/data/hg17/bed/geneSorter/blastp/dm1
cd /cluster/data/hg17/bed/geneSorter/blastp/dm1
mkdir run
cd run
mkdir out

# Make blast script
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
-p blastp -d /cluster/bluearc/dm1/blastp/flyBase \
-i $1 -o $2 -e 0.01 -m 8 -b 1
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy
# Create parasol batch
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...
# Completed: 7735 of 7735 jobs
# CPU time in finished jobs:      72141s    1202.35m    20.04h    0.83d  0.002 y
# IO & Wait Time:                 41717s     695.28m    11.59h    0.48d  0.001 y
# Average job time:                  15s       0.25m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              57s       0.95m     0.02h    0.00d
# Submission to last job:           204s       3.40m     0.06h    0.00d

# Load into database.  
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/dm1/run/out
hgLoadBlastTab hg17 dmBlastTab -maxPer=1 *.tab
# Scanning through 7735 files
# Loading database with 27109 rows

# update knownToHInv table
# Verified that there is now new release of HInv data.
hgMapToGene hg17 HInvGeneMrna knownGene knownToHInv
# count changed to 28851

# The new KG process no longer need entries in knownGeneLink (used to store
# info for DNA based RefSeqs.  So clean out the old data in knownGeneLink.

hgsql hg17 -e "delete from knownGeneLink"

#### RE-BUILD SUPERFAMILY RELATED TABLES (DONE - 2005-05-27 - Fan)

# Download latest Superfamily data files and build the Superfamily DB
# from supfam.mrc-lmb.cam.ac.uk

    mkdir /cluster/store10/superfamily/050524
    ln -s /cluster/store10/superfamily/050524 /cluster/data/superfamily/050524
    cd /cluster/data/superfamily/050524

# ftp over the following two files:
ass_22-May-2005.tab.gz
supfam_22-May-2005.sql.gz
    gzip -d *.gz

# Load the Superfamily database
    hgsql hg17 -e "create database superfam050524"
    nice hgsql superfam050524 < supfam_22-May-2005.sql &

# This may take about an hour.

# Make sure to add an index on id of the des table of superfam050524.
    hgsql superfam050524 -e "create index id on des(id);"

    hgsql superfam050524 < ~/src/hg/lib/sfAssign.sql
    hgsql superfam050524 -e 'load data local infile "ass_22-May-2005.tab" into table 
superfam050524.sfAssign;'

# Build or rebuild Superfamily track and create sf tables needed for PB

   hgsql hg17 < ~/src/hg/lib/sfAssign.sql

   cd /cluster/data/superfamily/050524  
   hgsql hg17 -e 'load data local infile "ass_22-May-2005.tab" into table hg17.sfAssign;'

# If hg17.sfDes already exists, drop it.

   hgsql superfam050524 -N -e "select * from des" >sfDes.tab
   hgsql hg17 < ~/src/hg/lib/sfDes.sql
   hgsql hg17 -e 'load data local infile "sfDes.tab" into table sfDes'

# If hg17.superfamily already exists, drop it.
   cd /cluster/data/hg17/bed
   mkdir /cluster/data/hg17/sf.2004-1128
   ln -s sf.2004-1128 sf
   hgSuperfam hg17 > sf.log

# It is normal that many proteins does not have corresponding Superfamily entries.

# If hg17.sfDescription exists, drop it.

   hgsql hg17 < ~/src/hg/lib/sfDescription.sql
   hgsql hg17 -e 'LOAD DATA local INFILE "sfDescription.tab"  into table hg17.sfDescription;'

# Finally, load the superfamily table.

   hgLoadBed hg17 superfamily superfamily.tab -tab

# Create knownToSuperfamily table
# Note hs is changed into ht for this Superfamily release.
   
   cat /cluster/data/superfamily/050524/ass_22-May-2005.tab \
   | hgKnownToSuper hg17 hs stdin
# created 32906 rows in knownToSuper

# Build tables needed by pbGlobal in proteins050415

   cd /cluster/data/superfamily/050524  
   hgsql proteins050415 -e 'load data local infile "ass_22-May-2005.tab" into table sfAssign'

   hgsql proteins050415 -e 'load data local infile "sfDes.tab" into table sfDes'

   cd /cluster/store10/kg/kgHg17F
   hgsql proteins050415 -e 'load data local infile "ensemblXref.tab" into table ensemblXref'

# These sf tables and ensemblXref3 are needed for non-HMR KG proteins.
# Should add content of ensemblXref3 of mm6 after it is done.
# And similarly for rn4 and possibly for other non-HMR species.
# CCDS <-> knownGene mapping need to be updated (markd 2005-05-29)
# this should be part of the known gene build
    /cluster/data/genbank/bin/i386/mkCcdsGeneMap -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap

# Build targetScanS track - (DONE - 2005-06-22 Fan)
#       requested by: Dudekula, Dawood (NIH/NIA/IRP) DudekulaDB@grc.nia.nih.gov
    ssh hgwdev 
    mkdir -p /cluster/data/hg17/bed/targetScanS
    cd /cluster/data/hg17/bed/targetScanS
    wget --timestamp http://genes.mit.edu/targetscan/tracks/targetscan.bed

# Remove the first description line of targetscan.bed

    hgLoadBed -tab hg17 targetScanS targetscan.bed

# Create/edit/check in targetScans.html and trackDb.ra under
# kent/src/hg/makeDb/trackDb/human/hg17

# Update mrnaRefseq table (DONE - Fan 6/22/05)
# The old table contains non-human mrna/RefSeqs.
# The new table contains only human mrna/RefSeq and RefSeq/RefSeq.

# First build entrez DB tables, see makeMm6.doc for details.

    hgsql entrez -N -e \
    'select mrna, refseq from entrezRefseq, entrezMrna, hg17.all_mrna where entrezRefseq.geneID=entrezMrna.geneID and mrna=all_mrna.qName' \
    >mrnaRefseq1.tab

# Include RefSeq as valid mRNA too.
    hgsql hg17 -N -e 'select name, name from refGene' >mrnaRefseq2.tab

    cat mrnaRefseq1.tab mrnaRefseq2.tab |sort -u >mrnaRefseq.tab

    hgsql hg17 -e 'drop table mrnaRefseq'
    hgsql hg17 < ~/src/hg/lib/mrnaRefseq.sql
    hgsql hg17 -e 'load data local infile "mrnaRefseq.tab" into table mrnaRefseq'

# BUILD KNOWN GENE LIST FOR GOOGLE.  DONE 6/27/05 Fan.
# make knownGeneLists.html hg17GeneList.html mm5GeneList.html rm3GeneList.html

    cd /cluster/data/hg17/bed
    rm -rf knownGeneList/hg17

# Run hgKnownGeneList to generate the tree of HTML pages
# under ./knownGeneList/hg17

    hgKnownGeneList hg17

# copy over to /usr/local/apache/htdocs

    rm -rf /usr/local/apache/htdocs/knownGeneList/hg17
    mkdir -p /usr/local/apache/htdocs/knownGeneList/hg17
    cp -Rfp knownGeneList/hg17/* /usr/local/apache/htdocs/knownGeneList/hg17

####  Blat knownGene proteins to determine exons (DONE braney 06-30-05)
    ssh hgwdev
    cd /cluster/data/hg17/bed
    mkdir blat.hg17KG.2005-06-17
    rm blat.hg17KG
    ln -s  blat.hg17KG.2005-06-17 blat.hg17KG
    cd blat.hg17KG
    pepPredToFa hg17 knownGenePep known.fa
    hgPepPred hg17 generic blastKGPep02 known.fa
    grep ">" known.fa | sed "s/>//" > kgName.lst
    kgName hg17 kgName.lst blastKGRef02
    hgsql hg17 < ~/kent/src/hg/lib/blastRef.sql
    echo "rename table blastRef to blastKGRef02" | hgsql hg17
    echo "load data local infile 'blastKGRef02' into table blastKGRef02" | hgsql hg17
    ssh kk
    cd /cluster/data/hg17/bed/blat.hg17KG
    cat << '_EOF_' > blatSome
#!/bin/csh -fe
/cluster/bin/i386/blat -t=dnax -q=prot -out=pslx $1 $2 $3
'_EOF_'
    # << keep emacs happy
    chmod +x blatSome
    ls -1S /scratch/hg/gs.18/build35/bothMaskedNibs/*.nib > human.lst
    mkdir kgfa
    cd kgfa
    faSplit sequence ../known.fa 3020 kg
    cd ..
    ls -1S kgfa/*.fa > kg.lst
    cat << '_EOF_' > blatGsub
#LOOP
blatSome $(path1) {check in line $(path2)} {check out line psl/$(root1)/$(root2).psl}
#ENDLOOP
'_EOF_'
    # << keep emacs happy
    gensub2 human.lst kg.lst blatGsub blatSpec
    mkdir psl
    cd psl
    foreach i (`cat ../human.lst`)
	mkdir `basename $i .nib`
    end
    cd ..
    para create blatSpec
    para push
# Completed: 134320 of 134320 jobs
# CPU time in finished jobs:   22196680s  369944.67m  6165.74h  256.91d  0.704 y
# IO & Wait Time:               1712586s   28543.10m   475.72h   19.82d  0.054 y
# Average job time:                 178s       2.97m     0.05h    0.00d
# Longest finished job:            7691s     128.18m     2.14h    0.09d
# Submission to last job:        608750s   10145.83m   169.10h    7.05d

# Completed: 133676 of 133676 jobs
# CPU time in finished jobs:   29661130s  494352.16m  8239.20h  343.30d  0.941 y
# IO & Wait Time:               2181179s   36352.99m   605.88h   25.25d  0.069 y
# Average job time:                 238s       3.97m     0.07h    0.02d
# Longest job:                   105972s    1766.20m    29.44h    1.23d

    ssh eieio
    cd /cluster/data/hg17/bed/blat.hg17KG
    pslSort dirs raw.psl /tmp psl/*
    pslReps -nohead -minCover=0.9 -minAli=0.9 raw.psl cooked.psl /dev/null
    pslUniq cooked.psl hg17KG.psl
    pslxToFa hg17KG.psl hg17KG_ex.fa -liftTarget=genome.lft -liftQuery=protein.lft

    ssh hgwdev
    cd /cluster/data/hg17/bed/blat.hg17KG
    kgName hg17 hg17KG.psl blastKGRef02
    cut -f 10 hg17KG.psl > kgName.lst
    faSomeRecords known.fa kgName.lst hg17KG.fa
    hgPepPred hg17 generic blastKGPep02 hg17KG.fa
#end blat proteins
    
# MAKE Drosophila Proteins track (DONE 07-05-05 braney)
    ssh kk
    mkdir -p /cluster/data/hg17/bed/tblastn.dm2FB
    cd /cluster/data/hg17/bed/tblastn.dm2FB
    echo  /panasas/store/hg17/blastDb/*.nsq  | xargs ls -S | sed "s/\.nsq//"  > target.lst    
    mkdir fbfa
    # calculate a reasonable number of jobs 
    calc `wc /cluster/data/dm2/bed/blat.dm2FB/dm2FB.psl|awk "{print \\\$1}"`/\(264630/`wc target.lst| awk "{print \\\$1}"`\)
# 18929/(350000/5959) = 322.279746
    split -l 322 /cluster/data/dm2/bed/blat.dm2FB/dm2FB.psl fbfa/fb
    cd fbfa
    for i in *; do pslxToFa $i $i.fa; rm $i; done
    cd ..
    ls -1S fbfa/*.fa > fb.lst
    mkdir -p /cluster/bluearc/hg17/bed/tblastn.dm2FB/blastOut  
    ln -s /cluster/bluearc/hg17/bed/tblastn.dm2FB/blastOut  
    for i in `cat fb.lst`; do  mkdir blastOut/`basename $i .fa`; done
    tcsh
    cat << '_EOF_' > blastGsub
#LOOP
blastSome $(path1) {check in line $(path2)} {check out exists blastOut/$(root2)/q.$(root1).psl } 
#ENDLOOP
'_EOF_'
    cat << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/iscratch/i/blast/data
export BLASTMAT
g=`basename $2`
f=/tmp/`basename $3`.$g
for eVal in 0.01 0.001 0.0001 0.00001 0.000001 1E-09 1E-11
do
if /scratch/blast/blastall -M BLOSUM80 -m 0 -F no -e $eVal -p tblastn -d $1 -i $2 -o $f.8
then
        mv $f.8 $f.1
        break;
fi
done
if test -f  $f.1
then
if /cluster/bin/i386/blastToPsl $f.1 $f.2
then
	liftUp -nosort -type=".psl" -nohead $f.3 /cluster/data/hg17/jkStuff/subLiftAll.lft warn $f.2  
	liftUp -nosort -type=".psl" -nohead $f.4 /cluster/data/hg17/jkStuff/liftAll.lft warn $f.3  
        liftUp -isPtoG -nosort -type=".psl" -pslQ -nohead $3.tmp /cluster/data/dm2/bed/blat.dm2FB/protein.lft warn $f.4
        mv $3.tmp $3
        rm -f $f.1 $f.2 $f.3
        exit 0
    fi
fi
rm -f $f.1 $f.2 $3.tmp $f.3 $f.8
exit 1
'_EOF_'

    chmod +x blastSome
    gensub2 target.lst fb.lst blastGsub blastSpec

    ssh kk
    cd /cluster/data/hg17/bed/tblastn.dm2FB
    para create blastSpec
    para push

# Completed: 351581 of 351581 jobs
# CPU time in finished jobs:   30733031s  512217.19m  8536.95h  355.71d  0.975 y
# IO & Wait Time:               1035790s   17263.16m   287.72h   11.99d  0.033 y
# Average job time:                  90s       1.51m     0.03h    0.00d
# Longest finished job:             816s      13.60m     0.23h    0.01d
# Submission to last job:        135367s    2256.12m    37.60h    1.57d

    ssh kki
    cd /cluster/data/hg17/bed/tblastn.dm2FB
    tcsh
    
    cat << '_EOF_' > chainGsub
#LOOP
chainSome $(path1) $(path2)
#ENDLOOP
'_EOF_'

    cat << '_EOF_' > chainSome
(cd $1; cat $2.psl | simpleChain -prot -outPsl -maxGap=150000 stdin ../c.`basename $1`.psl)
'_EOF_'
    chmod +x chainSome

    ls -1dS `pwd`/blastOut/fb?? > chain.lst
    gensub2 chain.lst single chainGsub chainSpec

    para create chainSpec
    para push
 
# Completed: 2714 of 2714 jobs
# CPU time in finished jobs:     222508s    3708.46m    61.81h    2.58d  0.007 y
# IO & Wait Time:                 10577s     176.29m     2.94h    0.12d  0.000 y
# Average job time:                  86s       1.43m     0.02h    0.00d
# Longest finished job:            9787s     163.12m     2.72h    0.11d

    cd /cluster/data/hg17/bed/tblastn.dm2FB/blastOut
    for i in fb??
    do 
	awk "(\$13 - \$12)/\$11 > 0.6 {print}" c.$i.*.psl > c60.$i.psl
	sort -rn c60.$i.psl | pslUniq stdin u.$i.psl
	awk "((\$1 / \$11) ) > 0.60 { print   }" c60.$i.psl > m60.$i.psl
	echo $i
    done

    sort -u -T /tmp -k 14,14 -k 16,16n -k 17,17n u.*.psl m60* > /cluster/data/hg17/bed/tblastn.dm2FB/blastDm2FB.psl
    cd ..

    ssh hgwdev
    cd /cluster/data/hg17/bed/tblastn.dm2FB
    hgLoadPsl hg17 blastDm2FB.psl
    exit

    # back to kksilo
    rm -rf blastOut

# End tblastn

# Build kgReactome table for KG to Reactome xref.  Done 6/28/05 Fan.

    ssh hgwdev
    mkdir -p /cluster/store10/reactome/reactome050613
    rm    /cluster/data/reactome
    ln -s /cluster/store10/reactome/reactome050613 /cluster/data/reactome

    cd /cluster/data/reactome
    
    wget --timestamp http://www.reactome.org/download/current/sql.gz
    
    hgsql hg17 -e 'drop database reactome'
    hgsql hg17 -e 'create database reactome'
    zcat sql.gz| hgsql reactome 

    hgsql reactome -N -e 'select kgId, spID, DB_ID from ReferenceEntity, hg17.kgXref where identifier=spID' >kgReactome.tab;

    hgsql hg17 -e 'drop table kgReactome'
    hgsql hg17 < ~/src/hg/lib/kgReactome.sql
    hgsql hg17 -e 'load data local infile "kgReactome.tab" into table kgReactome'

# UPDATE WGRNA TRACK (DONE, 2005-07-05, Fan)

  ssh hgwdev
  cd /cluster/data/hg17/bed

  mv wgRna wgRna-2005-06-16
  mkdir wgRna-2005-07-05
  cd wgRna-2005-07-05

# Received the data file, wgtrack_july2005.txt, from Michel Weber's email (Michel.Weber@ibcg.biotoul.fr)
# and place it under cd /cluster/data/hg17/bed/wgRna-2005-07-05.

  cat wgtrack_july2005.txt|sed -e 's/ /\t/g' >wgRna.tab
  
# edit wgRna.tab to take out the first 5 lines of data field labels.

    hgLoadBed -sqlTable=/cluster/home/fanhsu/src/hg/lib/wgRna.sql hg17 wgRna wgRna.tab

# REBUILT knownToPfam TABLE TO ALLOW KG REPRESENTED BY VARIANT SPLICE PROTEINS MAPPED TO PFAM (DONE 7/14/05, Fan)
# hgMapViaSwissProt.c was updated to support this.

# Create table that maps between known genes and Pfam domains
hgMapViaSwissProt hg17 knownGene name proteinID Pfam knownToPfam


## EVOFOLD (DONE, 2005-07-15, Jakob (jsp) ) 
# EvoFold is a new comparative method for predicting functional RNA
# secondary structures based on multiple sequence alignnments. The
# predictions generated for the EvoFold track are based on the most
# conserved elements of the 8-way alignment (multiz8way). The current
# data is the result of a pilot study (ongoing research of mine), the
# procedure used to generate the data will therefore be simplified
# when forthcoming evofold tracks for other organism are made. The
# documentation therefore skips the actual data generation, and
# instead starts with a data file I provide.

  ssh -C hg17
  mkdir -p /cluster/data/hg17/bed/evofold
  cd /cluster/data/hg17/bed/evofold
  cp /cluster/home/jsp/data/rnass/genome-scan/vertebrate/folds_hg17.bed foldsHg17.bed
  # The folds_hg17.bed is a 9-column bed file: column 1-6 provide
  # standard bed information, column 7 is element length, column 8 is
  # the RNA secondary structure in parentheses format, and column nine
  # is a commaseparated list of position specific confidence scores
  # (floats).
  hgLoadBed -notItemRgb -sqlTable=/cluster/home/jsp/prog/kent/src/hg/lib/evofold.sql hg17 evofold foldsHg17.bed

##########################################################################
# TRANSFRAG PHASE 2 TABLES  - lifted from hg15 (Jakob Skou Pedersen)
# Done: July 21, 2005
#
# These tables were lifted for use in my own research, but may be used
# for the 'Affymetrix Transcriptome Project Phase 2' tracks.
  ssh -C hgwdev
  mkdir -p /cluster/data/hg17/bed/transfrag
  cd /cluster/data/hg17/bed/transfrag
  # lifting transfrag tables from hg15 via hg16 to hg17
  for name in A375CytosolicPolyAPlusTnFg FHs738LuCytosolicPolyAPlusTnFg HepG2CytosolicPolyAMinusTnFg HepG2CytosolicPolyAPlusTnFg HepG2NuclearPolyAMinusTnFg HepG2NuclearPolyAPlusTnFg JurkatCytosolicPolyAPlusTnFg NCCITCytosolicPolyAPlusTnFg PC3CytosolicPolyAPlusTnFg SKNASCytosolicPolyAPlusTnFg U87CytosolicPolyAPlusTnFg; do
    echo "select chrom, chromStart, chromEnd, name from ${name};" | hgsql hg15 | sed -e 1d > ${name}Hg15.bed
    liftOver ${name}Hg15.bed /cluster/data/hg15/bed/liftOver/hg15ToHg16.over.chain ${name}Hg16.bed unmappedHg16.bed
    liftOver ${name}Hg16.bed /cluster/data/hg16/bed/liftOver/hg16ToHg17.over.chain ${name}Hg17.bed unmappedHg17.bed
    echo "hg16 unmapped count for ${name}: " `grep "#" unmappedHg16.bed | wc -l | awk '{print $1}'`
    echo "hg17 unmapped count for ${name}: " `grep "#" unmappedHg17.bed | wc -l | awk '{print $1}'`
    hgLoadBed hg17 ${name} ${name}Hg17.bed
    # clean up
    rm ${name}Hg15.bed ${name}Hg16.bed unmappedHg16.bed unmappedHg17.bed
  done

# GLADSTONE ARRAY TRACK (DONE 7/19/2005 Andy)
    ssh hgwdev
    cd /cluster/data/hg17/bed
    mkdir gladHumES
    cd gladHumES/
    cp /cluster/data/hg16/bed/geneAtlas2/geneAtlas2.bed .
    cut -f1-12 geneAtlas2.bed > bed.hg16
    liftOver bed.hg16 /gbdb/hg16/liftOver/hg16ToHg17.over.chain bed.hg17 /dev/null
    hgMapMicroarray bed.hg17.data hgFixed.gladHumESRatio \
       -bedIn bed.hg17
#Loaded 11087 rows of expression data from hgFixed.gladHumESRatio
#Mapped 10925,  multiply-mapped 382, missed 23266, unmapped 162
    hgLoadBed hg17 gladHumES bed.hg17.data

# PHASTODDS GENESORTER COLUMN (DONE 7/28/2005 Andy)
    ssh kolossus
    cd /panasas/store/andy
    mkdir phastOdds
    cd phastOdds/
    export PATH=${PATH}:/cluster/bin/phast/x86_64
    mkdir sso beds
    cat > runChrom.sh << "_EOF_"
#!/bin/bash
c=$1
numDir=`echo ${c#chr} | sed 's/_random//'`
ALNDIR=/cluster/data/hg17/bed/multiz10way

echo msa_view $c
/cluster/bin/phast/x86_64/msa_view --in-format MAF ${ALNDIR}/maf/${c}.maf --refseq /cluster/data/hg17/${numDir}/${c}.fa > /tmp/${c}.sso

echo phastCons $c
/cluster/bin/phast/x86_64/phastOdds -f ${ALNDIR}/cons/run.elements/ave.cons.mod -b ${ALNDIR}/cons/run.elements/ave.noncons.mod -g kg/${c}.bed /tmp/${c}.sso > /tmp/${c}.phastOdds.gtf
cp /tmp/${c}.sso sso/
rm /tmp/${c}.sso
cp /tmp/${c}.phastOdds.gtf gtfs/
rm /tmp/${c}.phastOdds.gtf
echo $c done
_EOF_
    ssh hgwdev
    cd /panasas/store/andy/phastOdds
    genePredToGtf hg17 knownGene kg.gtf
    exit 
    for c in `cut -f1 kg.gtf | sort | uniq`; do 
       grep "\b${c}\b" kg.gtf > kg/${c}.gtf; 
    done
    for f in kg/*.bed; do
       c=`basename $f .bed`;
       echo $c;
       ./runChrom.sh $c;
       addPhastOddsExons $f gtfs/$c.phastOdds.gtf beds/$c.bed
    done
    cat beds/* | sort -k4,4 -k1,1 -k2,2n -k3,3n > phastOdds.kg.bed
    cat > phastOdds.sql << "EOF"
CREATE TABLE phastOdds (
    bin smallint not null, # Speedup.
    chrom varchar(255) not null,        # Human chromosome or FPC contig
    chromStart int unsigned not null,   # Start position in chromosome
    chromEnd int unsigned not null,     # End position in chromosome
    name varchar(255) not null, # Name of item
              #Indices
    score float not null, # phastOdds score.
    index(chrom(8),bin),
    index(name(10))
);
EOF
# <<
    hgLoadBed -sqlTable=phastOdds.sql hg17 phastOdds phastOdds.kg.bed
    # Actually I probably don't need that hg17 table.
    echo create table phastOdds select name, score from hg17.phastOdds | hgsql hgFixed
    echo create index nameIndex on phastOdds (name(10)) | hgsql hgFixed    

##########################################################################
# Illumina SNPs (Heather, July 2005)
# Source: Jeff Ohmen, PhD, johmen@illumina.com, 858/232-2702

# using /cluster/store11 because /cluster/data/hg17 is on store5,
# which is currently being restored

  cd /cluster/store11/heather/illumina
  fix.pl < LinkageIVbSNP.txt > illumina.bed
  hgLoadBed hg17 snpIllumina -tab -strict -sqlTable=snpIllumina.sql illumina.bed
  # Reading illumina.bed
  # Loaded 6008 elements of size 4
  # Sorted
  # Saving bed.tab
  # Loading hg17

  # note: 28 rows where chrom = "chrXY"

# reload rankProp and psiBlast gene sorter tables to link with new
# known genes (markd 2005-07-15)
   (spLoadRankProp -noMapFile=max1k.nomap hg17 rankProp --  /cluster/bluearc/markd/rankprop/results/hs.sw+tr/max1k.rankp.gz)   >&max1k.hg17.out
   (spLoadPsiBlast hg17 spPsiBlast  /cluster/bluearc/markd/rankprop/results/hs.sw+tr.eval.gz) >&pslBlast.hg17.out 


# BLASTZ/CHAIN/NET CANFAM2 (DONE 8/2/05 angie - REDONE 12/12/05 angie - REDONE 2/6/06 angie)
# Unfortunately, this was done with a corrupted 
# /san/sanvol1/scratch/hg17/nib/chr5.nib the first time around; 
# also, a linSpecRep bug in blastz-run-ucsc has been fixed since then.
# Doh, then Kate pointed out that linSpecReps were not snipped properly --
# I had omitted the BLASTZ_ABRIDGE_REPEATS line from the DEF!!!  
# Added an error message to doBlastzChainNet.pl to catch that next time.
# Therefore I'm moving aside the previous run:
    mv /usr/local/apache/htdocs/goldenPath/hg17/vsCanFam2{,.bak}
# And rerunning...
    ssh kkstore02
    mkdir /cluster/data/hg17/bed/blastz.canFam2.2006-02-06
    cd /cluster/data/hg17/bed/blastz.canFam2.2006-02-06
    cat << '_EOF_' > DEF
# human vs. dog

BLASTZ_ABRIDGE_REPEATS=1

# TARGET: Human
SEQ1_DIR=/san/sanvol1/scratch/hg17/nib
SEQ1_SMSK=/san/sanvol1/scratch/hg17/linSpecRep.notInDog
SEQ1_LEN=/cluster/data/hg17/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Dog
SEQ2_DIR=/san/sanvol1/scratch/canFam2/nib
SEQ2_SMSK=/san/sanvol1/scratch/canFam2/linSpecRep.notInHuman
SEQ2_LEN=/cluster/data/canFam2/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/blastz.canFam2.2006-02-06
'_EOF_'
    # << for emacs
    doBlastzChainNet.pl DEF -bigClusterHub pk -smallClusterHub pk \
      -workhorse pk \
      -blastzOutRoot /san/sanvol1/scratch/blastzHg17CanFam2Out >& do.log &
    tail -f do.log
    rm -f /cluster/data/hg17/bed/blastz.canFam2
    ln -s blastz.canFam2.2006-02-06 /cluster/data/hg17/bed/blastz.canFam2

# RE-RUN NETTOAXT, AXTTOMAF FOR CANFAM2 (DONE 10/26/05 angie)
    # Kate fixed netToAxt to avoid duplicated blocks, which is important 
    # for input to multiz.  Regenerate maf using commands from sub-script 
    # netChains.csh generated by doBlastzChainNet.pl above.  
# Obsoleted by re-run of hg17-canFam2 above 12/12/05 angie...
    ssh kolossus
    cd /cluster/data/hg17/bed/blastz.canFam2.2005-08-01/axtChain
    netSplit hg17.canFam2.net.gz net
    chainSplit chain hg17.canFam2.all.chain.gz
    cd ..
    mv axtNet axtNet.orig
    mkdir axtNet
    foreach f (axtChain/net/*.net)
      netToAxt $f axtChain/chain/$f:t:r.chain \
        /panasas/store/hg17/nib /iscratch/i/canFam2/nib stdout \
      | axtSort stdin stdout \
      | gzip -c > axtNet/$f:t:r.hg17.canFam2.net.axt.gz
    end
    rm -r mafNet
    mkdir mafNet
    foreach f (axtNet/*.hg17.canFam2.net.axt.gz)
      axtToMaf -tPrefix=hg17. -qPrefix=canFam2. $f \
            /cluster/data/hg17/chrom.sizes /cluster/data/canFam2/chrom.sizes \
            stdout \
      | gzip -c > mafNet/$f:t:r:r:r:r:r.maf.gz
    end
    rm -r axtChain/{chain,net}/ axtNet.orig


############
# Sangamo/EIO DNaseI Hypersensitive Sites (2005-08-15 kate)
#   (Sangamo Biosciences and European Inst. Oncology)
#   Contact: Fyodor Umov (fumov@sangamo.com)

    cd /cluster/data/hg17/bed
    mkdir sangamo
    cd sangamo
    grep chr 3314_hs_sites_browser.bed | grep -v browser | \
        hgLoadBed -noBin hg17 sangamoDnaseHs stdin
            # Loaded 3314 elements of size 6
    checkTableCoords -table=sangamoDnaseHs hg17

    # use "antiword" to create plain text from .doc description file

# UPDATE WGRNA TRACK (DONE, 2005-08-24, Fan)

  ssh hgwdev
  cd /cluster/data/hg17/bed

  mkdir wgRna-2005-08-24
  cd wgRna-2005-08-24

# Received the data file, wgtrack_aug2005.txt, from Michel Weber's email 
# (Michel.Weber@ibcg.biotoul.fr)
# and place it under cd /cluster/data/hg17/bed/wgRna-2005-08-24.

  cut -f 2-10 wgtrack_aug2005.txt >wgRna.tab
  vi wgRna.tab
  
# edit wgRna.tab to take out the first line of data field labels.

    hgLoadBed -sqlTable=/cluster/home/fanhsu/src/hg/lib/wgRna.sql hg17 wgRna wgRna.tab

# Compared to 7/5 data, one record updated, one record dropped, one record added, out of 741 records.

# Generate snpMask files (Done Heather Sept. 1, 2005)
# Takes about 10-15 minutes
# Consumes about 1 gig of disk
  ssh hgwdev
  cd /usr/local/apache/htdocs/goldenPath/hg17
  mkdir snpMask
  cd snpMask
  foreach chrom ( `cat /cluster/data/hg17/chrom.lst` )
      snpMaskChrom hg17 ${chrom} /gbdb/hg17/nib/${chrom}.nib ${chrom}.ambigs.fa
      gzip ${chrom}.ambigs.fa
  end

#############################################################################
# BLASTZ Mm7 (WORKING - 2005-09-06 - Hiram)
#	Experiment, try the alignments without the linage specific
#	repeats
    ssh pk
    mkdir /cluster/data/hg17/bed/blastzMm7.2005-09-06
    cd /cluster/data/hg17/bed
    ln -s blastzMm7.2005-09-06 blastz.mm7
    cd blastzMm7.2005-09-06
    cat << '_EOF_' > DEF
# human vs mouse
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/x86_64:/cluster/home/angie/schwartzbin

BLASTZ=blastz.v7.x86_64
BLASTZ_H=2000
BLASTZ_M=50
BLASTZ_ABRIDGE_REPEATS=0

# TARGET: Human Hg17
SEQ1_DIR=/cluster/bluearc/hg17/bothMaskedNibs
SEQ1_LEN=/cluster/bluearc/hg17/chrom.sizes
SEQ1_CTGDIR=/cluster/bluearc/hg17/bothMaskedNibs
SEQ1_CTGLEN=/cluster/bluearc/hg17/chrom.sizes
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=500000
SEQ1_LAP=50

# QUERY: Mouse Mm7
SEQ2_DIR=/cluster/bluearc/mm7/mm7.2bit
SEQ2_LEN=/cluster/bluearc/mm7/chrom.sizes
SEQ2_CTGDIR=/cluster/bluearc/mm7/mm7Chroms_RandomContigs.2bit
SEQ2_CTGLEN=/cluster/bluearc/mm7/mm7Chroms_RandomContigs.sizes
SEQ2_LIFT=/cluster/bluearc/mm7/Chroms_RandomContigs.lft
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=3000000000
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/blastzMm7.2005-09-06
TMPDIR=/scratch/tmp
'_EOF_'
    # happy emacs

    cp -p /cluster/data/hg17/chrom.sizes ./S1.len
    twoBitInfo /cluster/bluearc/mm7/mm7Chroms_RandomContigs.2bit S2.len

    #	establish a screen to control this job
    screen
    time ./doBlastzChainNet.pl -stop chainMerge \
	-bigClusterHub=pk \
	`pwd`/DEF > toChainMerge.run.out 2>&1 &
    #	STARTED - 2005-09-06 - 11:00
    #	detach from screen session: Ctrl-a Ctrl-d
    #	to reattach to this screen session:
    time $HOME/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk \
	-continue=cat -stop=cat \
	`pwd`/DEF > catStep.out 2>&1 &

    time $HOME/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk \
	-continue=chainRun \
	`pwd`/DEF > continueChainRun.out 2>&1 &

    time $HOME/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk \
	-continue=chainMerge -stop=chainMerge \
	`pwd`/DEF > chainMerge.out 2>&1 &
    time $HOME/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk \
	-continue=net -stop=net \
	`pwd`/DEF > net.out 2>&1 &

    time $HOME/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk \
	-continue=load -stop=load \
	`pwd`/DEF > load.out 2>&1 &

    time $HOME/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk \
	-swap -stop=load \
	`pwd`/DEF > swap.out 2>&1 &

    #	Create plain pslChrom files to load a simple blastz track
    ssh kkstore02
    cd /cluster/data/hg17/bed/blastzMm7.2005-09-06
    mkdir -p pslChrom
(cd pslParts; ls | awk -F"." '{print $1}' | sort -u) | while read C
do
    echo -n "working ${C} ... "
    zcat pslParts/${C}.nib*.gz | gzip -c > pslChrom/${C}.psl.gz
    echo "done"
done
    #	Load those alignments
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastzMm7.2005-09-06
ls pslChrom | sed -e "s/.psl.gz//" | while read T
do
echo "hgLoadPsl -fastLoad -noTNameIx hg17 -table=${T}_blastzMm7 pslChrom/${T}.psl.gz"
hgLoadPsl -fastLoad -noTNameIx hg17 -table=${T}_blastzMm7 pslChrom/${T}.psl.gz
done


    #	After this same alignment was done with Hg17 query and Mm7
    #	target, came back to these swapped results in mm7 and manually loaded
    #	the swapped tables as: chainMm7LSR, chainMm7LSRLink and
    #	netMm7LSR
    #	41,223,632 total rows in the chainMm7Link split tables
    #	58,458,613 total rows in the chainMm7LSRLink table

    time featureBits hg17 chainMm7LSRLink
    #	959444893 bases of 2866216770 (33.474%) in intersection
    #	real    36m30.822s
    #	user    14m19.620s
    #	sys     5m13.910s
    time featureBits hg17 chainMm7Link
    #	955168137 bases of 2866216770 (33.325%) in intersection
    #	real    16m13.902s
    #	user    10m20.780s
    #	sys     3m42.810s
    #	And, their intersection:
    ssh kolossus
    time HGDB_CONF=~/.hg.conf.read-only featureBits hg17 \
	chainMm7LSRLink chainMm7Link
    #	952667399 bases of 2866216770 (33.238%) in intersection
    #	real    38m53.448s
    #	user    8m38.853s
    #	sys     2m23.362s

# LOAD ACEMBLY TRACK (DONE, 2005-09-12, Fan)
    mv /cluster/data/hg17/bed/acembly /cluster/data/hg17/bed/acembly_050217
    mkdir -p /cluster/data/hg17/bed/acembly
    cd /cluster/data/hg17/bed/acembly
    # Data is obtained from Jean Thierry-Mieg     mieg@ncbi.nlm.nih.gov
 
    wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35g.human.genes/acembly.ncbi_35g.genes.proteins.fasta.tar.gz
    wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35g.human.genes/acembly.ncbi_35g.genes.gff.tar.gz
    wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35g.human.genes/acembly.ncbi_35g.mrnas.fasta.tar.gz
    wget --timestamping ftp://ftp.ncbi.nih.gov/repository/acedb/ncbi_35g.human.genes/acembly.ncbi_35g.genes.pfamhits.tar.gz
    tar xvzf acembly.ncbi_35g.genes.gff.tar.gz
    tar xvzf acembly.ncbi_35g.genes.proteins.fasta.tar.gz
    cd acembly.ncbi_35.genes.gff
    # the acembly dataset for hg16 had problems with reverse blocks so
    # check for these
cat << '_EOF_' > checkReversedBlocks
for i in x1*.gff
do
        echo -n "$i working ..."
        awk -F"\t" '
{
if ($4 > $5) {
    printf "reverse blocks problem for $1"
        printf "\n"
}
}
' $i > $i.fixed
        echo " done"
done
'_EOF_'
    # << this line makes emacs coloring happy
    chmod +x checkReversedBlocks
    ./checkReversedBlocks  
    ls -l *.fixed
    # all *.fixed files are empty so remove - there is no reversing of blocks
    rm *.fixed

    foreach f (x1.acemblygenes.*.gff)
      set c=$f:r:e
      egrep '^[a-zA-Z0-9]+\|NT_[0-9][0-9][0-9][0-9][0-9][0-9]' $f | \
        perl -wpe 's/^(\w+)\|(\w+)/$1\/$2/' > ctg-chr${c}_random.gff
      if (-e ../../../$c/lift/random.lft) then
        liftUp chr${c}_random.gff ../../../$c/lift/random.lft warn \
          ctg-chr${c}_random.gff
      endif
      grep -v ^$c\| $f | grep -v ^Hs | perl -wpe 's/^/chr/;' | \
        grep -v "^chr//" > chr$c.gff
      echo "done $c"
    end
    
    #- Load into database - use extended genePred
    ssh hgwdev
    cd /cluster/data/hg17/bed/acembly
    # Reloaded without -genePredExt 1/6/05:
    ldHgGene -gtf hg17 acembly acembly.ncbi_35.genes.gff/chr*.gff
    # for entry with 28212470 from chr6.gff, change to chr6 
    # and for 29124352 in chr6.gff, change to chr6 (1/13/05)
    echo 'update acembly set chrom = "chr6" where chrom = "chr28212470";' \
         | hgsql hg17 
    echo 'update acembly set chrom = "chr6" where chrom = "chr29124352";' \
         | hgsql hg17 
    
    # checkTableCoords and runGeneCheck to check data
    checkTableCoords hg17 acembly

hgPepPred hg17 generic acemblyPep \
        acembly.ncbi_35.genes.proteins.fasta/*.fasta

# create table of Acembly gene classifications
cd /cluster/data/hg17/bed/acembly/acembly.ncbi_35.genes.gff
rm acemblyClass.tab
foreach f (x1.acemblygenes.*.gff)

cut -f 9 $f |sed -e 's/;/\t/g' |sed -e 's/transcript_id //' >j.tmp
cut -f 2 j.tmp >j2.tmp
cut -f 3 j.tmp >j3.tmp
paste j3.tmp j2.tmp|sed -e 's/Main_gene/main/g' |sed -e 's/Putative_gene/putative/g' |sed -e 's/ //g' >>acemblyClass.tab
end

rm *.tmp
hgsql hg17 -e 'drop table acemblyClass'
hgsql hg17 < ~src/hg/lib/acemblyClass.sql
hgsql hg17 -e 'load data local infile "acemblyClass.tab" into table acemblClass.tab'
hgsql hg17 -e 'delete from acemblyClass where class!="main" and class!="putative"'

# build acemblyPep table

hgPepPred hg17 generic acemblyPep \
        acembly.ncbi_35.genes.proteins.fasta/*.fasta

# Please note, per email from Jean Thierry-Mieg on 9/9/05,
# there are AceView genes (~10,000) without corresponding 
# protein sequences.  They will fix it next time.

###########################################################################
# LOADING AFFYTXNPHASE2 TRACK (sugnet)

# cd to where data is downloaded.
cd /cluster/store10/sugnet/affyTranscription/graphs/transcriptome.affymetrix.com/download/publication/polyA_minus/graphs

# lift data from hg16 to hg17. This takes a long time.
./liftWigFilesHg16ToHg17.sh

# make the .wib and .wig files. This takes a long time.
./makeWibWigHg17.sh

# Copy .wib files to /cluster/data/hg17 files
mkdir /cluster/data/hg17/bed/affyTxnPhase2/wigData/
cp `find ./ -name "*.hg17.wib"` /cluster/data/hg17/bed/affyTxnPhase2/wigData/
chmod 775 /cluster/data/hg17/bed/affyTxnPhase2/wigData/
chmod 664 /cluster/data/hg17/bed/affyTxnPhase2/wigData/*

# Make gbdb entry
mkdir /gbdb/hg17/wib/affyTxnPhase2
chmod 775 /gbdb/hg17/wib/affyTxnPhase2
cd /gbdb/hg17/wib/affyTxnPhase2
ln -s /cluster/data/hg17/bed/affyTxnPhase2/wigData/* .
cd -

# Load the database tables (using bash) this takes a while
for file in `find ./ -name "*hg17.wig"`; do
  base=`basename $file .hg17.wig`
  echo "Doing ${base}Txn"
  hgLoadWiggle -pathPrefix=/gbdb/hg17/wib/affyTxnPhase2  hg17 ${base}Txn $file
done

# Do the transfrags
cd ../transfrags
./liftHg16ToHg17.sh
./loadHg17Tnfg.sh

# End of affyTxnPhase2

###########################################################################
#  Creating download files for the affyTxnPhase2 data
#		(DONE - 2006-11-20 - Hiram)
    #	Copy all of the data above to a temporary /san/sanvol1/scratch/
    #	location, and run the following script:
#!/bin/sh

mkdir -p rawData/hg17

TOP=`pwd`
export TOP
for dir in `find ./ -type d | grep '_' | grep -v A375_cytosolic_polyAPlus | grep -v FHs738Lu_cytosolic_polyAPlus | grep -v HepG2_CytosolVsNucleusDifferenceGraphs | grep -v HepG2_cytosolic_polyAPlus | grep -v HepG2_cytosolic_polyAMinus | sed -e "s#^./##"`; do
base=`echo $dir | sed -e 's/\.\///; s/\//_/g' | sed -e 's/polyA-/polyAMinus/g' | sed -e 's/-/_/g' | sed -e 's/\+/Plus/g' | $TOP/changeName.pl`
RAW=$TOP/rawData/hg17/$base.data
echo $RAW
   cd $dir;
   zcat `ls -1 *hg17.bed.gz` | bedSort stdin stdout | cut -f 1,2,3,4 | grep chr | $TOP/avgerizeBed.pl > $RAW
   cd $TOP;
done

    #	Then copy the rawData/hg17/ results directory back to:
    /cluster/data/hg17/bed/affyTxnPhase2/rawResults/
    #	And deliver to hgdownloads via symlinks on hgwdev:
    cd /usr/local/apache/htdocs/goldenPath/hg17/affyTxnPhase2/
    # to:
    ln -s /cluster/data/hg17/bed/affyTxnPhase2/rawData/*.data.gz .
    #	Remove the san scratch data

###########################################################################
# ALTGRAPHX TRACK (sugnet)

/cluster/store1/sugnet/altSplice
mkdir hg17-2005.03.28

# First get the RNA clusters
cd hg17-2005.03.28

# Don't use RAGE libraries for clone bounds.
~/kent/src/hg/geneBounds/clusterRna/generateRageAccList.csh  hg17 rage.libs

# Make spec file to run.
foreach c (`echo 'select chrom from chromInfo' | hgsql hg17 | grep -v chrom`)
    set out = chrom/$c.bed
    echo "clusterRna -mrnaExclude=hg17.rage.libs hg17 /dev/null $out -chrom=$c" >> clusterRna.spec
end

# Tried running it on the minicluster, but can't connect to the 
# cluster accounts so run it from here on hgwdev.
./clusterRna.spec >& clusterRna.log

cd ..

# Make script to setup parasol job file for raw altGraphX files on human
cat << '_EOF_' > makeRun.sh
#!/bin/sh

for chrom in `echo "select chrom from chromInfo" | hgsql hg17 | grep -v chrom`; do
echo 'echo "Doing $chrom"'
echo "/cluster/home/sugnet/bin/i386/altSplice -db=hg17   -beds=rnaCluster/chrom/$chrom.bed -agxOut=agxs/hg17.$chrom.agx -consensus -weightMrna -localMem -minAli=.95 -minCover=.5 -chromNib=/cluster/data/hg17/nib/$chrom.nib"
done
'_EOF_'
    # << this line makes emacs coloring happy
mkdir agxs
chmod 755 makeRun.sh

# Minicluster down, have to run on hgwdev.
./makeRun.sh > toRun.sh
chmod 755 toRun.sh
./toRun.sh >& toRun.log &

cat agxs/*.agx > hg17.agx

# make raw altGraphX files for mouse
mkdir ../mm5-2005.03.28/
cd ../mm5-2005.03.28/

# make the rnaClusters
mkdir rnaCluster
cd rnaCluster/
mkdir chrom

# Don't use RAGE libraries for clone bounds.
~/jk/hg/geneBounds/clusterRna/generateRageAccList.csh mm5 rage.libs
# Doing select on mm5 into mm5.rage.libs
# Done.

# Make spec file to run.
foreach c (`echo 'select chrom from chromInfo' | hgsql mm5 | grep -v chrom`)
    set out = chrom/$c.bed
    echo "clusterRna -mrnaExclude=mm5.rage.libs mm5 /dev/null $out -chrom=$c" >> clusterRna.spec
end

# Tried running it on the minicluster, but can't connect to the 
# cluster accounts so run it from here on hgwdev.
chmod 755 clusterRna.spec
./clusterRna.spec >& clusterRna.log &


# Make the gene bounds in rnaCluster.
mkdir agxs

# This script generates the jobs, one per chromosome.
echo << '_EOF_' > makeRun.sh
#!/bin/sh

for chrom in `echo "select chrom from chromInfo" | hgsql mm5 | grep -v chrom`; do
echo 'echo "Doing $chrom"'
echo "/cluster/home/sugnet/bin/i386/altSplice -db=mm5   -beds=rnaCluster/chrom/$chrom.bed -agxOut=agxs/mm5.$chrom.agx -consensus -weightMrna -localMem -minAli=.95 -minCover=.5 -chromNib=/cluster/data/mm5/nib/$chrom.nib"
done
'_EOF_'
chmod 755 makeRun.sh
./makeRun.sh > toRun.sh
chmod 755 toRun.sh
./toRun.sh >& toRun.log & # Takes an hour or so...

# Consolodiate all of the records in a single file.
cat agxs/*.agx > mm5.agx


# Make the orthologous splicing graphs.
mkdir orthoSpliceExoniphy
cd orthoSpliceExoniphy/

# Get the exoniphy exons...
echo "select chrom, txStart, txEnd, name, id, strand from exoniphy order by chrom, txStart;" | hgsql hg17 | grep -v txStart > hg17.exoniphy.bed

# Set up the commands for the orthosplice run.
echo 'select chrom, size from chromInfo' | hgsql hg17 | grep -v chrom > chromSizes.tab
ln -s /cluster/data/hg17/bed/blastz.mm5/axtChain/mouseNet/ nets
ln -s /cluster/data/hg17/bed/blastz.mm5/axtChain/chain/ chains
mkdir agx report logs

cat << '_EOF_' > makeRun.sh

#!/usr/bin/perl -w

open(IN, 'chromSizes.tab') or die "Can't open chromSizes.tab\n";
while(<IN>) {
    chomp;
    @w = split;
    print "/cluster/home/sugnet/bin/i386/orthoSplice -chromSize=$w[1] -exonFile=hg17.exoniphy.bed -trumpNum=3 -chrom=$w[0] -altInFile=../agxs/hg17.$w[0].agx -orthoAgxFile=../../mm5-2005.03.28/mm5.agx -db=hg17 -orthoDb=mm5 -netFile=nets/$w[0].net -chainFile=chains/$w[0].chain -commonFile=agx/$w[0].hg17.mm5.cons.t3.agx -reportFile=report/$w[0].hg17.report -edgeFile=report/$w[0].hg17.edge.report >& logs/$w[0].test.log\n";
}
'_EOF_'
# << emacs

./makeRun.sh > orthoSplice.para.spec
ssh kki
cd /cluster/store1/sugnet/altSplice/hg17-2005.03.28/orthoSpliceExoniphy
para create orthoSplice.para.spec
para push

cat agx/*.agx > hg17.mm5.t3.exoniphy.agx

# Make bed file
agxToBed hg17.mm5.t3.exoniphy.agx hg17.mm5.t3.exoniphy.bed

# Load up files
hgLoadBed hg17 agxBed hg17.mm5.t3.exoniphy.bed
hgLoadBed -notItemRgb -sqlTable=/cluster/home/sugnet/kent/src/hg/lib/altGraphX.sql hg17 altGraphX hg17.mm5.t3.exoniphy.agx

# end altGraphX track

# EXONWALK TRACK (sugnet)

# make altGraphX track (see above)

cd /cluster/store1/sugnet/altSplice/hg17-2005.03.28/orthoSpliceExoniphy

cd exonWalk
mkdir beds

# Make parasol script.
foreach file (`ls ../agx/*.agx`)
  set base=`basename $file .agx`
  echo "/cluster/home/sugnet/bin/i386/exonWalk db=hg17 minPercent=0 trumpSize=100000 $file beds/$base.bed" >> exonWalk.para.spec
end
para create exonWalk.para.spec
para push
cat beds/*.bed > hg17.mm5.cons.t3.exoniphy.bed

# Predict orfs
mkdir orfs
cd orfs
mkdir bedOrf beds fa borf
cp ~/store1/altSplice/hg17-2005.01.09/orthoSpliceExonify/exonWalk/orfs.mrna2/*.sh ./
splitFile ../../hg17.mm5.cons.t3.exoniphy.bed 500 exonWalk.
cat < < '_EOF_' > makeFa.sh
#!/bin/sh

for file in "$@"
do
 base=`basename $file`
 echo "Doing $file"
 echo "sequenceForBed -db=hg17 -bedIn=$file -fastaOut=fa/$base.fa "
 sequenceForBed -db=hg17 -bedIn=$file -fastaOut=fa/$base.fa 
done
'_EOF_'
chmod 755 makeFa.sh
makeFa.sh beds/*

# Run borf lots of times...
makeSpec.sh beds/* > para.spec
para create para.spec
para push
mkdir genePred
cat << '_EOF_' > makeGenePred.sh
#!/bin/sh

for file in "$@"
do
  base=`basename $file`
  /cluster/home/sugnet/bin/i386/borfMatcher -keepNmd beds/$base borf/$base.borf bedOrf/$base.bed genePred/$base.gp 
done
'_EOF_'
    # << this line makes emacs coloring happy
chmod 755 makeGenePred.sh 
makeGenePred.sh beds/*

cat beds/* > hg17.mm5.exonWalk.bed
cat genePred/*.gp > hg17.mm5.exonWalk.gp

wc *.bed *.gp
# 155470 1865640 29956585 hg17.mm5.exonWalk.bed
#  98433  984330 32495119 hg17.mm5.exonWalk.gp

# Load it into the database.
ldHgGene -predTab hg17 exonWalk hg17.mm5.exonWalk.gp 

# end exonWalk

####################################################################
### hapmapRecombRate (Daryl; September 19, 2005)
    Lifted from hg16; see makeHg16.doc for details
    # Update (Jen; October 25, 2005)
    Data points that lifted to chroms other than 1-22 + X removed
    before release to RR (confirmed with Daryl).
    chr4_random: 11 data points
    chr6_hla_hap1: 25 data points
### hapmapRecombHotspot (Daryl; September 19, 2005)
    Lifted from hg16; see makeHg16.doc for details

### HapMap SNPs (Daryl; February 4, 2006)
  # most of this work was done in October and November 2005 for the ENCODE workshop
    cd /cluster/store4/gs.17/build34/bed/hapmap/frequencies/2005-10/non-redundant/hapmapSnps
    ln -sf ../hg17.daf.all/daf.txt.gz .
    ln -sf ../hg17.panTro1.rheMac1.txt.gz .
    zcat hg17.panTro1.rheMac1.txt | grep -v chrom | sort >! hg17.panTro1.rheMac1.sort.txt
    zcat daf.txt                  | grep -v chrom | sort >! daf.sort.txt
    # check that order matches; should be empty
    paste hg17.panTro1.rheMac1.sort.txt daf.sort.txt | awk '$1!=$17||$2!=$18||$3!=$19||$4!=$20||$5!=$21||$6!=$22||$7!=$23||$8!=$24||$11!=$25||$12!=$27||$15!=$26||$16!=$28{print $0;}'
    paste hg17.panTro1.rheMac1.sort.txt daf.sort.txt | awk '{printf "%s\t%d\t%d\t%s\t0\t%c\t%c\t%c\t%c\t%c\t%d\t%d\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\n",$1,$2,$3,$4,$6,$7,$8,$11,$15,$12,$16,$29,$30,$31,$32,$33}' >! hapmapSnps.bed
    hgLoadBed hg17 hapmapSnps -sqlTable=hapmapSnps.sql hapmapSnps.bed

############################################################################################
# HapMap SNPs rel21a (Feb. 2007, Heather)
# June 2007 [partial fix of hapmapAllelesSummary released 6/25/07:
# using hg17 instead of hg18 liftOver files... for most but not all
# chroms! :( not documented below; error found by user]
# 1/11/08, 1/24/08 (angie): regenerated hapmapAllelesSummary with corrected
# hapmapAllelesChimp.

# get files for each chrom, for each population
# these contain data for all individuals
# not using the JPT+CHB files
ssh kkstore05
cd /cluster/store12/snp/hapmap/rel21a/genotypes/2007-01/rs_strand/non-redundant/zips
wget http://www.hapmap.org/downloads/genotypes/2007-01/rs_strand/non-redundant/* 

# get population data (needed to define children in CEU and YRI trios)
cd /cluster/store12/snp/hapmap
wget http://www.hapmap.org/downloads/samples_individuals/*gz
gunzip pedinfo2sample_CEU.txt.gz
filterPedigree.pl < pedinfo2sample_CEU.txt > CEU.filtered
cp CEU.filtered rel21a/genotypes/2007-01/rs_strand/non-redundant/CEU.list
gunzip pedinfo2sample_YRI.txt.gz
filterPedigree.pl < pedinfo2sample_YRI.txt > YRI.filtered
cp YRI.filtered rel21a/genotypes/2007-01/rs_strand/non-redundant/YRI.list

#!/usr/bin/env perl
while (<STDIN>) {
    my @fields = split;
    if ($fields[2] == 0 && $fields[3] == 0) {
        @subfields = split /:/, $fields[6];
        print $subfields[4];
        print "\n";
    }
}

cd /cluster/store12/snp/hapmap/rel21a/genotypes/2007-01/rs_strand/non-redundant
zcat zips/*chr22_CEU* | head -1 > header.CEU
zcat zips/*chr22_YRI* | head -1 > header.YRI
# add carriage returns to header.CEU and header.YRI
grep -n -f CEU.list header.CEU > offsets.CEU
grep -n -f YRI.list header.YRI > offsets.YRI
# delete ids in offsets.CEU and offsets.YRI so just column numbers remain

# for each population, combine all chroms, and combine all individuals
# for CEU and YRI, filter out children from trios
# This creates CEU.merge, CHB.merge, JPT.merge, YRI.merge

./merge.csh

#!/bin/tcsh 
rm -f CEU.merge
rm -f CHB.merge
rm -f JPT.merge
rm -f YRI.merge

foreach chrom (`cat chrom.list`)

    echo $chrom

    # CEU
    echo "CEU"
    set fileName=`echo $chrom | awk '{printf "zips/genotypes_%s_CEU_r21a_nr.txt.gz", $1}'`
    zcat $fileName | filterCEU.pl >> CEU.merge

    # CHB
    echo "CHB"
    set fileName=`echo $chrom | awk '{printf "zips/genotypes_%s_CHB_r21a_nr.txt.gz", $1}'`
    zcat $fileName | filterCHB.pl >> CHB.merge

    # JPT
    echo "JPT"
    set fileName=`echo $chrom | awk '{printf "zips/genotypes_%s_JPT_r21a_nr.txt.gz", $1}'`
    zcat $fileName | filterJPT.pl >> JPT.merge

    # YRI
    echo "YRI"
    set fileName=`echo $chrom | awk '{printf "zips/genotypes_%s_YRI_r21a_nr.txt.gz", $1}'`
    zcat $fileName | filterYRI.pl >> YRI.merge

end


# Below is filterCEU.pl
# The others are very similar: YRI uses "offsets.YRI"
# CHB and JPT just read the input directly

#!/usr/bin/env perl 

# read in a list of the columns that we are keeping
sub initList {
open LIST, "offsets.CEU";
chomp(@list = <LIST>);
close LIST;
$listSize = @list;
}

&initList;

while (<STDIN>) {
    my @fields = split;
    # skip header
    if ($fields[0] eq "rs#") { next; }
    # chrom
    print $fields[2];
    print " ";
    # position: add zero-based start coord
    print $fields[3] - 1;
    print " ";
    print $fields[3];
    print " ";
    # rsId
    print $fields[0];
    print " ";
    # score
    print "0 ";
    # strand
    print $fields[4];
    print " ";
    # observed
    print $fields[1];
    print " ";
    @alleles = ();
    for ( my $loop = 0; $loop < $listSize; $loop++ ) {
	push (@alleles, $fields[@list[$loop]-1]);
    }
    # N is used for missing data
    $nCount = 0;
    # counts
    $aCountHomo = 0;
    $cCountHomo = 0;
    $gCountHomo = 0;
    $tCountHomo = 0;
    $aCountHetero = 0;
    $cCountHetero = 0;
    $gCountHetero = 0;
    $tCountHetero = 0;
    foreach $allele (@alleles) {
        $parent1 = substr($allele, 0, 1);
        $parent2 = substr($allele, 1, 1);
	# Ns must be together
	if ($parent1 eq "N" && $parent2 ne "N") { die "Unexpected input"; }
	if ($parent2 eq "N" && $parent1 ne "N") { die "Unexpected input"; }
	if ($parent1 eq "N" && $parent2 eq "N") { $nCount++; next; }

	if ($parent1 eq "A" && $parent2 eq "A") { 
	    $aCountHomo = $aCountHomo + 2;
	    next; 
	}
	if ($parent1 eq "C" && $parent2 eq "C") { 
	    $cCountHomo = $cCountHomo + 2;
	    next; 
	}
	if ($parent1 eq "G" && $parent2 eq "G") { 
	    $gCountHomo = $gCountHomo + 2;
	    next; 
	}
	if ($parent1 eq "T" && $parent2 eq "T") { 
	    $tCountHomo = $tCountHomo + 2;
	    next; 
	}

	if ($parent1 eq "A") { $aCountHetero++; }
	if ($parent1 eq "C") { $cCountHetero++; }
	if ($parent1 eq "G") { $gCountHetero++; }
	if ($parent1 eq "T") { $tCountHetero++; }
	if ($parent2 eq "A") { $aCountHetero++; }
	if ($parent2 eq "C") { $cCountHetero++; }
	if ($parent2 eq "G") { $gCountHetero++; }
	if ($parent2 eq "T") { $tCountHetero++; }
    }
    print "A ";
    print $aCountHomo;
    print " ";
    print $aCountHetero;
    print " ";
    print "C ";
    print $cCountHomo;
    print " ";
    print $cCountHetero;
    print " ";
    print "G ";
    print $gCountHomo;
    print " ";
    print $gCountHetero;
    print " ";
    print "T ";
    print $tCountHomo;
    print " ";
    print $tCountHetero;
    print " ";

    print "\n";

}

# << emacs
# Switch to C programs from kent/src/hg/snp/snpLoad.

# Determine allele1 and allele2 (set allele2 to "none" if monomorphic)
# Alleles are in alphabetical order
# Calculate score (minor allele frequency)
# Log and skip if wrong number of elements in row
# Log and skip if triallelic or quadallelic
# Log and skip degenerate case (no alleles) 
# No errors this run
# Still running on kkstore05
# Could rename "hapmap1" to "hapmapCondense"
/cluster/home/heather/kent/src/hg/snp/snpLoad/hapmap1 CEU.merge CEU.condense
wc -l hapmap1.log
/cluster/home/heather/kent/src/hg/snp/snpLoad/hapmap1 CHB.merge CHB.condense
wc -l hapmap1.log
/cluster/home/heather/kent/src/hg/snp/snpLoad/hapmap1 JPT.merge JPT.condense
wc -l hapmap1.log
/cluster/home/heather/kent/src/hg/snp/snpLoad/hapmap1 YRI.merge YRI.condense
wc -l hapmap1.log

# save some space
gzip *merge

# load
ssh hgwdev
cd /cluster/store12/snp/hapmap/rel21a/genotypes/2007-01/rs_strand/non-redundant
cp /cluster/home/heather/kent/src/hg/lib/hapmapSnps.sql .
# modify hapmapSnps for 4 populations
hgLoadBed hg17 hapmapSnpsCEU -sqlTable=hapmapSnpsCEU.sql CEU.condense
hgLoadBed hg17 hapmapSnpsCHB -sqlTable=hapmapSnpsCHB.sql CHB.condense
hgLoadBed hg17 hapmapSnpsJPT -sqlTable=hapmapSnpsJPT.sql JPT.condense
hgLoadBed hg17 hapmapSnpsYRI -sqlTable=hapmapSnpsYRI.sql YRI.condense

# save some more space
ssh kkstore05
cd /cluster/store12/snp/hapmap/rel21a/genotypes/2007-01/rs_strand/non-redundant
gzip *condense

# sanity check
mysql> select count(*) from hapmapSnpsCEU where homoCount1 + homoCount2 + heteroCount = 0;
+----------+
| count(*) |
+----------+
|        0 |
+----------+

mysql> select count(*) from hapmapSnpsCHB where homoCount1 + homoCount2 + heteroCount = 0;
+----------+
| count(*) |
+----------+
|        0 |
+----------+

mysql> select count(*) from hapmapSnpsJPT where homoCount1 + homoCount2 + heteroCount = 0;
+----------+
| count(*) |
+----------+
|        0 |
+----------+

mysql> select count(*) from hapmapSnpsYRI where homoCount1 + homoCount2 + heteroCount = 0;
+----------+
| count(*) |
+----------+
|        0 |
+----------+

mysql> select max(score) from hapmapSnpsCEU;  
+------------+
| max(score) |
+------------+
|        500 |
+------------+


# create indexes
mysql> alter table hapmapSnpsCEU add index name (name);
mysql> alter table hapmapSnpsCEU add index chrom (chrom, bin);
mysql> alter table hapmapSnpsCHB add index name (name);
mysql> alter table hapmapSnpsCHB add index chrom (chrom, bin);
mysql> alter table hapmapSnpsJPT add index name (name);
mysql> alter table hapmapSnpsJPT add index chrom (chrom, bin);
mysql> alter table hapmapSnpsYRI add index name (name);
mysql> alter table hapmapSnpsYRI add index chrom (chrom, bin);

# 2nd step in processing: create hapmapSnpsCombined
ssh hgwdev
cd /cluster/data/hg17/bed/hapmap/rel21a
/cluster/home/heather/kent/src/hg/snp/snpLoad/hapmap2 hg17
hgLoadBed hg17 hapmapSnpsCombined -sqlTable=/cluster/home/heather/kent/src/hg/lib/hapmapSnpsCombined.sql hapmapSnpsCombined.tab

# create indexes (not used by browser)
mysql> alter table hapmapSnpsCombined add index name (name);
mysql> alter table hapmapSnpsCombined add index chrom (chrom, bin);

# errors
# nothing that isn't biallelic
# nothing with mixed positions
# over 500K that were not available in all 4 populations
# YRI: 187,485
# CEU: 129,359
# CHB and JPT: 97,095

# Also, 2 strand corrections done
grep -v missing hapmap2.errors
# different strands for rs1621378
# different strands for rs5768

# cleanup to save space
rm hapmapSnpsCombined.tab

# monomorphism

YRI        867,835 
CEU      1,252,743 
CHB      1,496,438 
JPT      1,539,094 
combined   607,393 

# observed strings
# why is A/T different from other transversions?

A/G      1,344,043
C/T      1,344,542
A/C        352,875
A/T        275,670
C/G        354,299
G/T        354,149
triallelic   1,370
quadallelic    403
other        1,226

# some details on the others:
125     -/A/T
124     -/A/G
107     -/C/T
85      -/A/C
79      -/G/T
25      -/C/G

18      -/A/C/T
13      -/A/G/T
12      -/A/C/G
11      -/C/G/T

7       (LARGEINSERTION)
5       (LARGEDELETION)
6       microsat
2       het

# check for collisions (more than one SNP at the same location)
# none found
ssh hgwdev
cd /cluster/data/hg17/bed/hapmap/rel21a
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpCheckCluster2 hg17 hapmapSnpsCombined > snpCheckCluster2.out

# check against hg17.snp125
ssh hgwdev
cd /cluster/data/hg17/bed/hapmap/rel21a
/cluster/home/heather/kent/src/hg/snp/snpLoad/hapmapLookup hg17 hapmapSnpsCombined snp125 snp125Exceptions

# 1817 total that are complex type from dbSNP (hapmapLookup.log)
# This is not based on observed string, only on size, class and locType
# 1176 class = mixed
#  616 class = single but locType != exact
#   11 class = named
#    6 class = insertion
#    4 class = deletion
#    2 class = microsat
#    2 class = het

# Generally if class = single the observed string is bi-allelic as expected
# Exceptions to that:
#    rs700519 quad-allelic, locType = rangeDeletion
#    rs1572672 tri-allelic, locType = between
#    rs2357412 tri-allelic, locType = range
#    rs2364671 tri-allelic, locType = rangeSubstitution
#    rs3959788 quad-allelic, locType = between

# 74 items in hapmapLookup.error
# 59 reverse complement (that's okay)
#  7 multiple alignment (6 from chrX:154,219,000-154,220,500 which is close to PAR)
# Also rs6645103 which is PAR 
mysql> select chrom, chromStart, chromEnd, strand, observed, class, locType, weight from snp125 where name = "rs6645103";
+-------------+------------+----------+--------+----------+--------+---------+--------+
| chrom       | chromStart | chromEnd | strand | observed | class  | locType | weight |
+-------------+------------+----------+--------+----------+--------+---------+--------+
| chrX_random |     273788 |   273789 | -      | C/T      | single | exact   |      3 |
| chrX        |     421141 |   421142 | +      | C/T      | single | exact   |      3 |
| chrY        |     421141 |   421142 | +      | C/T      | single | exact   |      3 |
+-------------+------------+----------+--------+----------+--------+---------+--------+


#  4 observed with dbSNP complex, hapmap biallelic
#  all positive strand, locType = between
#  all cluster errors in dbSNP
#  rs10485830
#  rs7625205 (intronic)
#  rs713582
#  rs11403115 (class = insertion)

# 3 observed mismatch
# all dbSNP clustering error
# rs2230624 (tri-allelic)
# rs3963317 (monomorphic in hapmap, rangeSubstitution in dbSNP)
# rs5017503 (monomoprhic in hapmap)

# a strange one
# rs731449
# dbSNP strand = -, hapmap strand = +
# dbSNP observed = G/T, hapmap observed = C/T
# dbSNP clustering error rs2321451, which is C/T
# hapmap monomorphic for T
# ortho A
# no repeats, no genes, no mRNAs, no conservation

# Counts of rows where 3 populations have one major allele, the 4th has the other

hapmapMixed hg17
# countCEU = 162931
# countCHB = 46543
# countJPT = 48791
# countYRI = 309105

# Generate summary table (used by filters)
# Summary table includes ortho allele and ortho qual score
# Summary table score is heterozygosity
# Individual zygosity is *not* preserved
ssh hgwdev
# 6/25/08: regenerated with mostly-corrected hapmapAllelesChimp
# 1/11/08 angie: regenerated with finally-corrected (I hope) hapmapAllelesChimp
# 1/24/08 angie: regenerated with finally-corrected (I hope!) hapmapAllelesChimp
cd /cluster/data/hg17/bed/hapmap/rel21a
/cluster/home/heather/kent/src/hg/snp/snpLoad/hapmapSummary hg17 hapmapSnpsCombined hapmapAllelesChimp hapmapAllelesMacaque
hgLoadBed hg17 hapmapAllelesSummary -sqlTable=/cluster/home/heather/kent/src/hg/lib/hapmapAllelesSummary.sql hapmapSummary.tab -tab

# sanity check
mysql> select count(*) from hapmapAllelesSummary where majorAlleleCountCEU > totalAlleleCountCEU;
+----------+
| count(*) |
+----------+
|        0 |
+----------+

mysql> select count(*) from hapmapAllelesSummary where majorAlleleCountCHB > totalAlleleCountCHB;
+----------+
| count(*) |
+----------+
|        0 |
+----------+

mysql> select max(score) from hapmapAllelesSummary;
+------------+
| max(score) |
+------------+
|        500 |
+------------+

mysql> select count(*), popCount from hapmapAllelesSummary group by popCount;
+----------+----------+
| count(*) | popCount |
+----------+----------+
|    52479 |        1 |
|    72977 |        2 |
|   207643 |        3 |
|  3700478 |        4 |
+----------+----------+

mysql> select count(*), isMixed from hapmapAllelesSummary group by isMixed; 

+----------+---------+
| count(*) | isMixed |
+----------+---------+
|  3192896 | NO      |
|   840681 | YES     |
+----------+---------+


# histogram of heterozygosity:

  0 ************************************************************ 883400
 25 ************** 204000
 50 ************* 188703
 75 *********** 157404
100 ********** 143119
125 ********* 131575
150 ********* 126916
175 ********* 128585
200 ******** 123440
225 ******** 119815
250 ******** 120646
275 ******** 120239
300 ******** 122654
325 ********* 128233
350 ********* 130069
375 ********** 144699
400 ********** 152829
425 ************ 172513
450 *************** 225645
475 ********************************** 503166
500  5927

############################################################################################

### HapMap LD (Daryl; February 11, 2006)
    ## start from the genotypes files, run Haploview, reformat, and load
    mkdir -p /san/sanvol1/hg17/bed/hapmap/genotypes/2006-01/non-redundant/para
    cd /san/sanvol1/hg17/bed/hapmap/genotypes/2006-01/non-redundant
    # wget all genotype data:
    # ftp://www.hapmap.org/genotypes/2006-01/non-redundant/genotypes_chr*_*.b35.txt.gz

    # Haploview had to be recompiled because there was a missing JPT sample in the ped file
    ##runHaploview.csh
    #!/bin/csh
    if ( $#argv < 2 ) then
	echo "usage: $0 <absolutePath> <genotypeFileName.gz> [<javaMaxMem>]"
	echo "       $0 /cluster/bin/foo bar.gz 2G"
	exit 1
    endif
    set path = $1
    set file = $2
    set root = $file:r
    set memFlag = ""
    if ( $#argv >= 3  ) then
	set memFlag = "-Xmx$3"
    endif
    cd /scratch
    /bin/cp     -f $path/$file .
    /bin/gunzip -f $file
    /usr/java/jre1.5.0_06/bin/java -d64 $memFlag -jar /cluster/home/daryl/haploview/haploview/Haploview.jar -c -d -n -maxDistance 250 -a $root >&! $root.log
    /bin/gzip   -f $root.LD    $root.CHECK >>& $root.log
    /bin/mv     -f $root.LD.gz $root.CHECK.gz  $root.log $path/
    /bin/rm     -f $root*
    ###

    cd para
    set hv = /cluster/home/daryl/scripts/runHaploview.csh
    set ldDir = /cluster/store5/gs.18/build35/bed/hapmap/genotypes/2006-01/non-redundant
    
    foreach pop (YRI CEU CHB JPT JPT+CHB)
	foreach chrom (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X)
	    echo $hv $ldDir genotypes_chr${chrom}_{$pop}.b35.txt.gz 4G >> jobList
	end
    end
    ssh pk
    # para create, para try, para push -maxNode=25 ...
    #Completed: 120 of 120 jobs
    #CPU time in finished jobs:    1564915s   26081.91m   434.70h   18.11d  0.050 y
    #IO & Wait Time:                 21862s     364.37m     6.07h    0.25d  0.001 y
    #Average job time:               13223s     220.39m     3.67h    0.15d
    #Longest running job:                0s       0.00m     0.00h    0.00d
    #Longest finished job:           40742s     679.03m    11.32h    0.47d
    #Submission to last job:        104809s    1746.82m    29.11h    1.21d

    #### makeDcc.pl 
    #!/usr/bin/perl -W
    $pop   = shift || die "usage: makeDcc.pl <pop> <chr>\n";
    $chrom = shift || die "usage: makeDcc.pl <pop> <chr>\n";
    $geno  = "geno/genotypes_${chrom}_${pop}.b35.txt.gz";
    $ld    = "ld/genotypes_${chrom}_${pop}.b35.txt.LD.gz";
    $txt   = "dcc/ld_${chrom}_${pop}.b35.txt";
    open(GENO,"zcat $geno | " ) || die "can't open $geno";
    open(LD,  "zcat $ld   | " ) || die "can't open $ld";
    open(TXT, " > $txt      " ) || die "can't open $txt";
    <GENO>;#ignore header
    while (<GENO>) { @fields = split / /; $pos{$fields[0]} = $fields[3]; }
    close(GENO);
    <LD>;#ignore header;
    while (<LD>) { @fields     = split /\t/; $chromStart = $pos{$fields[0]}; $chromEnd   = $pos{$fields[1]};
	print TXT "$chromStart $chromEnd $pop $fields[0] $fields[1] $fields[2] $fields[4] $fields[3]\n"; }
    close(LD);
    close(TXT);
    system("gzip $txt");
    ####

    #### makeDcc.csh
    #!/bin/csh
    #set path = "/cluster/home/daryl/scripts";
    set path = ".";
    foreach pop (CEU CHB JPT YRI JPT+CHB)
	foreach chr (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y)
	    echo $path/makeDcc.pl $pop chr$chr
	end
    end
    ####

    #### makeLdBed.pl
    #!/usr/bin/perl -W
    sub min ($$)
    {
	my $a = shift @_;
	my $b = shift @_;
	if ($a<$b) {return $a;}
	return $b;
    }
    sub encodeDprime($)
    {
	my $val = shift @_;
	if    ( ($val > 1) || ($val < -1) ) { die "Dprime value ($val) is out of range [-1,1]";}
	elsif ($val>=0) { $ret = ord('a') + $val*9;}
	else         { $ret = ord('A') - $val*9;}
	return chr($ret);
    }
    sub encodeRsquared($)
    {
	my $val = shift @_;
	if ( ($val > 1) || ($val < 0) ) { die "R^2 value ($val) is out of range [0,1]";}
	return encodeDprime($val);
    }
    sub encodeLod($$)
    {
	my $lod    = shift @_;
	my $dPrime = shift @_;
	$ret = ord('a');
	if ($lod>=2) # high LOD
	{
	if (abs($dPrime)<0.5) { $ret = ord('y'); } # high LOD, low D'  -> pink
	else { $ret += min((int($lod-abs($dPrime)-1.5)), 9) ;}
	}
	elsif (abs($dPrime)>0.99) { $ret = ord('z'); } # high D', low LOD  -> blue
	return chr($ret); 
    }
    $inDir  = shift||"data"; 
    $outDir = shift||"bed";
    $foo    = "";
    $bar    = ""; 
    @rest   = (); 
    @pops   = ("CEU", "CHB", "JPT", "YRI", "JPT+CHB");
    printf("> Starting        \t" . `date` . "\n");
    foreach $pop (@pops)
    {
	opendir(DIR, $inDir) || die "can't open $inDir";
	if ($pop eq "JPT+CHB") { @hmFiles = grep {/^ld_/ && /_JPT/ && /CHB.b35.txt.gz$/} readdir(DIR); }
	else { @hmFiles = grep {/^ld_/ && /_${pop}.b35.txt.gz$/} readdir(DIR); }
	closedir(DIR);
	printf "POP:\t$pop\t$#hmFiles\n";
	foreach $hmFile (sort @hmFiles)
	{
	    ($foo, $chrom, $bar) = split /_/, $hmFile;
	    $chrom =~ s/chrx/chrX/;
	    $chrom =~ s/chry/chrY/;
	    $outfile = "$outDir/${pop}_${chrom}.hg17.bed";
	    if ((-e $outfile)||(-e "$outfile.gz")) { next; }
	    $tmpFile = "/tmp/${pop}_${chrom}.hg17.bed";
	    printf("$inDir/$hmFile => $outfile.gz\t" . `date`);
	    open(OUT, "> $tmpFile" ) || die "can't open $tmpFile";
	    open(IN, "zcat $inDir/$hmFile | " ) || die "can't open $inDir/$hmFile";
	    $line = <IN>;
	    if (!defined $line){next;}
	    chomp($line);
	    ($chromStart, $chromEnd, $pop, $name, $marker2, $dprime, $rsquared, $lod, @rest) = split / /, $line;
	    $ldCount = 1;
	    while (<IN>)
	    {
	        chomp();
		($chromStartNew, $chromEndNew, $pop, $nameNew, $marker2, $dprime, $rsquared, $lod, @rest) = split / /;
		if ($chromStart ne $chromStartNew)
		{
		    $chromStart--;
		    printf(OUT "$chrom\t$chromStart\t$chromEnd\t$name\t$ldCount\t$dprimeList\t$rsquaredList\t$lodList\n");
		    $chromStart   = $chromStartNew;
		    $chromEnd     = $chromEndNew;
		    $name         = $nameNew;
		    $ldCount      = 1;
		    $dprimeList   = encodeDprime($dprime);
		    $rsquaredList = encodeRsquared($rsquared);
		    $lodList      = encodeLod($lod, $dprime);
		}
		elsif ($chromEndNew-$chromStartNew<250000)
		{
		    $chromEnd     = $chromEndNew;
		    $ldCount++;
		    $dprimeList   .= encodeDprime($dprime);
		    $rsquaredList .= encodeRsquared($rsquared);
		    $lodList      .= encodeLod($lod, $dprime);
		}
	    }
	    close(IN);
	    $chromStart--;
	    printf(OUT "$chrom\t$chromStart\t$chromEnd\t$name\t$ldCount\t$dprimeList\t$rsquaredList\t$lodList\n");
	    close(OUT);
	    system("gzip $tmpFile");
	    system("mv $tmpFile.gz $outDir");
	}
    }
    printf("> Finished        \t" . `date` . "\n");
    ####

    #### getMax.csh -- check for consistency by chrom and population
    #!/bin/csh
    set out = maxDist.txt
    rm -f $out
    touch $out
    echo this takes about 4 hours to run completely >> $out
    foreach f (dcc/ld_*.b35.txt.gz)
	echo -n "$f    " >> $out
	zcat $f | awk '{if ($2-$1>max) max=$2-$1} END {print max}' >> $out
    end

    #### getSizes.csh -- should all be 249999
    #!/bin/csh
    set out = wcList.txt
    rm -f $out
    touch $out
    echo "this takes about 2 hours to run completely"
    foreach f (dcc/*.txt.gz) 
	echo -n $f:r:r "    " | sed 's/ld_//;s/chr//;s/_/\t/' >> $out
	zcat $f | cut -f1 -d " " | uniq | wc -l >> $out
    end

    #### load.csh
    #!/bin/csh
    set db = hg17
    sed 's/hapmapLd/hapmapLdCeu/'    ${HOME}/kent/src/hg/lib/hapmapLd.sql | hgsql ${db}
    sed 's/hapmapLd/hapmapLdChb/'    ${HOME}/kent/src/hg/lib/hapmapLd.sql | hgsql ${db}
    sed 's/hapmapLd/hapmapLdJpt/'    ${HOME}/kent/src/hg/lib/hapmapLd.sql | hgsql ${db}
    sed 's/hapmapLd/hapmapLdYri/'    ${HOME}/kent/src/hg/lib/hapmapLd.sql | hgsql ${db}
    sed 's/hapmapLd/hapmapLdChbJpt/' ${HOME}/kent/src/hg/lib/hapmapLd.sql | hgsql ${db}
    # about half an hour to an hour per population
    foreach c (1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X)
	hgLoadBed -noSort -oldTable -strict ${db} hapmapLdCeu CEU_chr${c}.${db}.bed.gz
	hgLoadBed -noSort -oldTable -strict ${db} hapmapLdChb CHB_chr${c}.${db}.bed.gz
	hgLoadBed -noSort -oldTable -strict ${db} hapmapLdJpt JPT_chr${c}.${db}.bed.gz
	hgLoadBed -noSort -oldTable -strict ${db} hapmapLdYri YRI_chr${c}.${db}.bed.gz
	hgLoadBed -noSort -oldTable -strict ${db} hapmapLdChbJpt JPT+CHB_chr${c}.${db}.bed.gz
    end
    rm -f bed.tab
    ###


# AFFYHUEX1 TRACK (sugnet Wed Oct  5 12:16:42 PDT 2005)

mkdir hg17
cd hg17
pwd
# /cluster/store1/sugnet/affymetrixHumanAllExon/hg17
mkdir gff beds annot
cd gff
# download gff design files

# parse gff script...
#!/usr/bin/perl -w
if(scalar(@ARGV) == 0) {
    print STDERR "parseGff.pl - Parse out affymetrixes gff annotation 
probesets for human all exon design.
usage:
   parseGff.pl file1.design.gff file2.design.gff ... fileN.design.gff
";
    exit(1);
}

sub splitField($) {
    my $l = shift(@_);
    my @w = split / /, $l;
    return $w[1];
}

while($file = shift(@ARGV)) {
    if(!($file =~ /(.+)\.gff/)) {
	die "$file doesn't have .gff suffix\n";
    }
    $prefix = $1;
    print STDERR "Doing file $file.\n";
    open(IN, $file) or die "Can't open $file to read.";
    open(BED, ">../beds/$prefix.pset.bed") or die "Can't open ../beds/$prefix.pset.bed to write.";
    open(ANNOT, ">../annot/$prefix.tab") or die "Can't open ../annot/$prefix.tab to write.";
    while($line = <IN>) {
	# Only want the probeset records.
	if($line =~ /\tprobeset\t/) {
	    $score = 0;
	    $cds = 0;
	    $bounded = 0;
	    chomp($line);
	    # pop off an microsoft line endings.
	    $line =~ s/\r$//;
	    @words = split /\t/, $line;
	    # This makes the evidence be comman separated.
	    $words[8] =~ s/\" \"/,/g;
	    # This gets rid of pesky quotes.
	    $words[8] =~ s/\"//g;

	    # Set the score based on the annotation type
	    if($words[8] =~ /full/) {
		$score = 200;
	    }
	    elsif($words[8] =~ /extended/) {
		$score = 500;
	    }
	    elsif($words[8] =~ /core/) {
		$score = 900;
	    }
	    if($words[8] =~ /bounded/) {
		$score -= 200;
	    }
	    if($words[8] =~ /cds/) {
		$score += 100;
	    }
	    if($score <= 0) {
		$score = 100;
	    }
		
	    # Print out the annotation fields.
	    @fields = split /; /,$words[8];
	    $id = splitField($fields[1]);
	    $f = shift(@fields);
	    $f = splitField($f);
	    print ANNOT "$f";
	    while($f = shift(@fields)) {
		if($f =~ /^bounded/) {
		    $bounded = 1;
		}
		if($f =~ /^cds/) {
		    $cds = 1;
		}
		if(!($f =~ /^bounded/ || $f =~ /^cds/)) {
		    $f = splitField($f);
		    print ANNOT "\t$f";
		}
	    }
	    print ANNOT "\t$bounded\t$cds";
	    print ANNOT "\n";
	    print BED "$words[0]\t$words[3]\t$words[4]\t$id\t$score\t$words[6]\n";
	}
    }
    close(IN);
    close(BED);
    close(ANNOT);
}

./parseGff.pl *.gff
cat beds/*.bed > affyHuEx1.bed
hgLoadBed hg17 affyHuEx1 affyHuEx1.bed -strict
cat annot/*.tab > affyHuEx1.annot.tab 

# Contents of affyHuEx1Annot.sql file
CREATE TABLE affyHuEx1Annot (
  numIndependentProbes smallint not null,
  probesetId int(11) not null,
  exonClustId int(11) not null,
  numNonOverlapProbes smallint not null,
  probeCount smallint not null,
  transcriptClustId int(11) not null,
  probesetType smallint not null,
  numXHybeProbe smallint not null,
  psrId int(11) not null,
  level varchar(10) not null,
  evidence varchar(255) not null,
  bounded smallint not null,
  cds smallint not null,
  PRIMARY KEY (probesetId)
);
hg17S -A < affyHuEx1Annot.sql 
echo "load data local infile 'affyHuEx1.annot.tab' into table affyHuEx1Annot;" | hg17S -A

# end AFFYHUEX1 track

##########################################################################
# AFFY HUEX1 OFF-BY-ONE FIX (Andy 2006-12-14)
   ssh hgwdev
   cd /cluster/data/hg17/bed/affyHumanExon
   echo "select * from affyHuEx1" | hgsql hg17 | \
      tail +2 | awk 'BEGIN{OFS="\t"}{print $2,$3-1,$4,$5,$6,$7}' \
    > affyHuEx1.fixed.bed
   hgLoadBed hg17 affyHuEx1 affyHuEx1.fixed.bed

##########################################################################
# NSCAN composite track - (2005-09-29 markd)  loaded proteins 2005-10-13
    cd /cluster/data/hg17/bed/nscan/
    # obtained NSCAN and NSCAN-EST predictions from michael brent's group
    # at WUSTL
    wget http://genome.cse.wustl.edu/predictions/human/hg17_nscan_mm5_9_14_2005/hg17_nscan_mm5_9_14_2005.tar.gz
    tar -zxf hg17_nscan_mm5_9_14_2005.tar.gz 

    wget http://genome.cse.wustl.edu/predictions/human/NCBI35_NSCAN_EST_4-16-2005.tar  
    gzip -9 NCBI35_NSCAN_EST_4-16-2005.tar

    # change protein fasta file to have transcript id in header
    foreach f (chr_ptx/*.ptx)
        awk '/^>/{$0=$1".a"}{print $0}' $f >$f.fix
    end
    foreach f (NCBI35_NSCAN_EST_4-16-2005/chr_ptx/*.ptx)
        awk '/^>/{$0=$1".a"}{print $0}' $f >$f.fix
    end

    # load tracks.  Note that these have *utr features, rather than
    # exon featres.  currently ldHgGene creates separate genePred exons
    # for these.
    ldHgGene -gtf -genePredExt hg17 nscanGene  chr_gtf/chr*.gtf
    hgPepPred hg17 generic nscanPep chr_ptx/chr*.fix
    rm -rf chr_* *.tab

    ldHgGene -gtf -genePredExt hg17 nscanEstGene NCBI35_NSCAN_EST_4-16-2005/chr_gtf/chr*.gtf
    hgPepPred hg17 generic nscanEstPep NCBI35_NSCAN_EST_4-16-2005/chr_ptx/chr*.fix
    rm -rf NCBI35_NSCAN_EST_4-16-2005 *.tab

    # update trackDb; need a hg17-specific page to describe informants
    human/hg17/nscan.html    
    human/hg17/trackDb.ra

##########################################################################
# NHGRI DNASE I HYPERSENSITIVE SITES (2005-10-05 kate)
#  Submitted by Greg Crawford via web site,
#  http://research.nhgri.nih.gov/DNaseHS/May2005/
#  In addition, a file containing the 'randoms' was FTP'ed by Greg
#  Submitted for hg16 -- lifted to hg17.
#  Details of hg16 data prep are in makeHg16.doc

    mkdir /cluster/data/hg17/bed/nhgri
    cd /cluster/data/hg17/bed/nhgri
    cp /cluster/data/hg16/bed/nhgri/hs.bed hs.hg16.bed
    liftOver hs.hg16.bed /gbdb/hg16/liftOver/hg16ToHg17.over.chain \
                hs.hg17.bed hs.unmapped
    grep '^chr' hs.unmapped | wc -l
        # 8 unmapped
    hgLoadBed hg17 nhgriDnaseHs hs.hg17.bed
        # Loaded 14216 elements of size 5
    checkTableCoords hg17 nhgriDnaseHs

# UPDATE WGRNA TRACK (DONE, 2005-10-20, Fan)

  ssh hgwdev
  cd /cluster/data/hg17/bed

  mkdir wgRna-2005-10-20
  cd wgRna-2005-10-20

# Received the data file, wgtrack_no_bin_oct2005.txt, from Michel Weber's email 
# (Michel.Weber@ibcg.biotoul.fr)
# and place it under cd /cluster/data/hg17/bed/wgRna-2005-10-20.

  cp wgtrack_no_bin_oct2005.txt wgRna.tab
  vi wgRna.tab
  
# edit wgRna.tab to take out the first line of data field labels.

    hgLoadBed -sqlTable=/cluster/home/fanhsu/src/hg/lib/wgRna.sql hg17 wgRna wgRna.tab

# Compared to 8/24/05 data, a few records were changed.

##########################################################################
# REBUILD hg17.gnfAtlas2Distance TABLE.  SOMEHOW IT HAD MUCH FEWER RECORDS. (DONE 10/27/05, Fan)

# Create table to map between known genes and GNF Atlas2
# expression data.
    hgMapToGene hg17 gnfAtlas2 knownGene knownToGnfAtlas2 '-type=bed 12'

# Create expression distance table - takes about an hour
    hgExpDistance hg17 hgFixed.gnfHumanAtlas2MedianRatio \
        hgFixed.gnfHumanAtlas2MedianExps gnfAtlas2Distance \
        -lookup=knownToGnfAtlas2 &

#       hgsql -e "select count(*) from gnfAtlas2Distance;" hg17
#       row count changed to 32458000
			
##########################################################################
# BUILD ALLEN BRAIN TRACK (DONE 10/29/05 JK)

# Make the working directory
    ssh hgwdev
    cd /cluster/data/hg17/bed
    mkdir allenBrain
    cd allenBrain

# Remap the probe alignments from mm7 to hg17
    zcat /cluster/data/mm7/bed/bedOver/mm7.hg17.over.chain.gz \
        |  pslMap -chainMapFile -swapMap \
	       /cluster/data/mm7/bed/allenBrain/allenBrainAli.psl stdin stdout
	|  sort -k 14,14 -k 16,16n > unscored.psl
    pslRecalcMatch unscored.psl /cluster/data/hg17/nib \
        /cluster/data/mm7/bed/allenBrain/allProbes.fa allenBrainAli.psl
    

# Load the database
   hgsql hg17 < ~/kent/src/hg/lib/allenBrainUrl.sql
   hgsql hg17 -e 'load data local infile "/cluster/data/mm7/bed/allenBrain/allenBrainUrl.tab" into table allenBrainUrl;'
   hgLoadPsl hg17 allenBrainAli.psl
   mkdir /gbdb/hg17/allenBrain
   ln -s /cluster/data/mm7/bed/allenBrain/allProbes.fa /gbdb/hg17/allenBrain/allProbes.fa
   hgLoadSeq hg17 /gbdb/hg17/allenBrain/allProbes.fa

# Make mapping between known genes and allenBrain	
   hgMapToGene hg17 allenBrainAli -type=psl knownGene knownToAllenBrain 

##########################################################################
# BUILD NIBB IMAGE PROGES (DONE 11/07/05 JK)

# Make directory on san for cluster job and copy in sequence
    ssh pk
    mkdir /san/sanvol1/scratch/hg17/nibbPics
    cd /san/sanvol1/scratch/hg17/nibbPics
    cp /cluster/data/xenTro1/bed/nibbPics/nibbImageProbes.fa .

# Make parasol job dir and sequence list files
    mkdir run
    cd run
    mkdir psl
    ls -1 /cluster/sanvol1/scratch/hg17/nib/*.nib > genome.lst
    echo ../nibbImageProbes.fa > rna.lst

# Create parasol gensub file file
cat << '_EOF_' > gsub
#LOOP
blatz -rna -minScore=6000 -out=psl $(path1) $(path2) psl/$(root1)_$(root2).psl
#ENDLOOP
'_EOF_'

# Create parasol batch
    gensub2 genome.lst mrna.lst gsub spec
    para create spec

# Do para try/push/time etc.
#Completed: 46 of 46 jobs
#CPU time in finished jobs:      11818s     196.97m     3.28h    0.14d  0.000 y
#IO & Wait Time:                   145s       2.41m     0.04h    0.00d  0.000 y
#Average job time:                 260s       4.33m     0.07h    0.00d
#Longest running job:                0s       0.00m     0.00h    0.00d
#Longest finished job:            1022s      17.03m     0.28h    0.01d
#Submission to last job:          1060s      17.67m     0.29h    0.01d

# Make sort and filter
    catDir psl | sort -k 10 \
        | pslReps stdin stdout /dev/null -nohead -minAli=0.60 -nearTop=0.001 -minCover=0.10 -minNearTopSize=80 \
	| sort -k 14,14 -k 16,16n \
	| sed 's/..\/..\/nib\/chr/chr/' \
	| sed 's/.nib//' > ../nibbImageProbes.psl

# Make bed file and copy in stuff
    ssh hgwdev
    cd /cluster/data/hg17/bed
    mkdir nibbPics
    cd nibbPics
    cp /cluster/data/xenTro1/bed/nibbPics/nibbImageProbes.fa .
    cp /san/sanvol1/scratch/hg17/nibbPics/nibbImageProbes.psl .

# Load into database
    ln -s /cluster/data/xenTro1/bed/nibbPics/nibbImageProbes.fa /gbdb/hg17/nibbImageProbes.fa
    hgLoadSeq hg17 /gbdb/hg17/nibbImageProbes.fa
    hgLoadPsl hg17 nibbImageProbes.psl

###########################################################################
# EXONIPHY WITH DOG (acs, 11/22/05) -- MM7, RN3, CANFAM2, HG17

    # first build 4-way multiz alignment from syntenic nets (helps reduce
    # false positive predictions due to paralogous alignments)

    # (prepare mafNet files from syntenic nets and copy to
    # /cluster/bluearc/hg17/mafNetSyn; do this for mm7, rn3, canFam2,
    # and galGal2)

    # make output dir and run dir
    ssh pk
    cd /cluster/data/hg17/bed/multiz.hg17Mm7Rn3CanFam2

    mkdir -p mafSyn runSyn
    cd runSyn

    # create scripts to run multiz on cluster
    cat > oneMultiz.csh << 'EOF'
#!/bin/csh -fe
    set c = $1
    set multi = /scratch/$user/multiz.hg17Mm7Rn3CanFam2.$c
    set pairs = /cluster/bluearc/hg17/mafNetSyn

    # special mode --
    # with 1 arg, cleanup
    if ($#argv == 1) then
        rm -fr $multi
        exit
    endif

    #  special mode --
    # with 3 args, saves an alignment file
    if ($#argv == 3) then
        cp $multi/$2/$c.maf $3
        exit
    endif 
        
    set s1 = $2
    set s2 = $3
    set flag = $4

    # locate input files -- in pairwise dir, or multiple dir
    set d1 = $multi
    set d2 = $multi
    if (-d $pairs/$s1) then
        set d1 = $pairs
    endif
    if (-d $pairs/$s2) then
        set d2 = $pairs
    endif
    set f1 = $d1/$s1/$c.maf
    set f2 = $d2/$s2/$c.maf
    # write to output dir
    set out = $multi/${s1}${s2}
    mkdir -p $out

    # check for empty input file
    if (-s $f1 && -s $f2) then
        echo "Aligning $f1 $f2 $flag"
        /cluster/bin/penn/multiz.v10.5 $f1 $f2 $flag > $out/$c.tmp.maf
        echo "Ordering $c.maf"
        /cluster/bin/penn/maf_project $out/$c.tmp.maf hg17.$c > $out/$c.maf
    else if (-s $f1) then
        cp $f1 $out
    else if (-s $f2) then
        cp $f2 $out
    endif
'EOF'
# << for emacs
    chmod +x oneMultiz.csh

    cat > allMultiz.csh << 'EOF'
#!/bin/csh -fe
set c = $1
oneMultiz.csh $c mm7 rn3 0
oneMultiz.csh $c mm7rn3 canFam2 1
# get final alignment file
oneMultiz.csh $c mm7rn3canFam2 /cluster/data/hg17/bed/multiz.hg17Mm7Rn3CanFam2/mafSyn/$c.maf
#cleanup
oneMultiz.csh $c
'EOF'
# << for emacs
    chmod +x allMultiz.csh

cat > gsub << 'EOF'
#LOOP
allMultiz.csh $(root1) {check out line+ /cluster/data/hg17/bed/multiz.hg17Mm7Rn3CanFam2/mafSyn/$(root1).maf}
#ENDLOOP
'EOF'
# << for emacs
    cut -f 1 /cluster/data/hg17/chrom.sizes > chrom.lst
    set path = (/parasol/bin $path);rehash
    gensub2 chrom.lst single gsub jobList
    para create jobList
        # 46 jobs
    para try; para check
    para push

    # build chromosome-by-chromosome SS files

    cd /cluster/data/hg17/bed/multiz.hg17Mm7Rn3CanFam2
    mkdir run-ss-syn
    cd run-ss-syn
    mkdir -p /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSyn
    cat > makeSS.csh << 'EOF'
#!/bin/csh -fe
set c = $1
/cluster/bin/phast/msa_view -i MAF -o SS /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/mafSyn/$c.maf --refseq /cluster/bluearc/hg17/chrom/$c.fa | gzip -c >  /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSyn/$c.ss.gz
'EOF'
# << for emacs
    chmod +x makeSS.csh

    rm -f jobList
    foreach chr (`cut -f 1 /cluster/data/hg17/chrom.sizes`)
	echo "makeSS.csh $chr" >> jobList
    end

    para create jobList
        # 46 jobs
    para try; para check
    para push

    # now train hmm, with indel model
    # note: commands below require bash

    # first get a clean set of genes for training (with --indel-strict)
    mkdir -p /cluster/data/hg17/bed/exoniphy/train
    cd /cluster/data/hg17/bed/exoniphy/train
    mkdir -p stats genes
    CHROMS="chr1 chr2 chr3 chr4 chr5 chr6 chr7 chr8 chr9 chr10 chr11 chr12 chr13 chr14 chr15 chr16 chr17 chr18 chr19 chr20 chr21 chr22"
    for chr in ${CHROMS} ; do 
	echo $chr 
	zcat /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSyn/$chr.ss.gz | clean_genes genes/refGene.$chr.gff - --stats stats/$chr.stats --conserved --indel-strict --groupby exon_id --offset3 4 --offset5 4 > genes/refGene.$chr.clean.gff 
    done

    # get conserved noncoding seqs and add to GFFs
    mkdir -p cns
    for chr in ${CHROMS} ; do 
	    echo $chr
	    featureBits -bed=cns/$chr.bed -chrom=$chr hg17 phastConsElementsPaper \!knownGene:exon:100 \!refGene:exon:100 \!mrna \!ensGene \!intronEst \!twinscan
	cp genes/refGene.$chr.clean.gff genes/refGene.$chr.withCNS.gff 
	awk '{printf "%s\tphastCons\tCNS\t%d\t%d\t.\t.\t.\texon_id \"CNS.%s\"\n", $1, $2+1, $3, $4}' cns/$chr.bed >> genes/refGene.$chr.withCNS.gff 
    done

    # now train HMM
    # note: actually have to unzip SS files before this step
    rm -f alns gffs
    for chr in ${CHROMS} ; do 
	echo /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSyn/$chr.ss >> alns 
	echo genes/refGene.$chr.withCNS.gff >> gffs 
    done
    hmm_train -m '*alns' -c ~/phast/data/exoniphy/default.cm -g '*gffs' -R exon_id -i SS -I CDS,background,CNS,5\'splice,3\'splice,prestart -t "((hg17,(mm7,rn3)),canFam2)" > indels.hmm

    # training complete; now run exoniphy genome-wide

    # first need to split up alignments
    
    mkdir -p /cluster/data/hg17/bed/exoniphy/test/run-split
    cd /cluster/data/hg17/bed/exoniphy/test/run-split
    
    cat > doSplit.csh << 'EOF'
#!/bin/csh -fe
set c = $1
mkdir -p /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSynFrags/$c
/cluster/bin/phast/msa_split /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/mafSyn/$c.maf --refseq /cluster/bluearc/hg17/chrom/$c.fa -i MAF --windows 100000,0 --between-blocks 5000 --min-informative 1000 --out-format SS --out-root /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSynFrags/$c/$c --tuple-size 3
gzip /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSynFrags/$c/$c*.ss
'EOF'
# << for emacs
    chmod +x doSplit.csh

    rm -f jobList
    for file in /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/mafSyn/chr*.maf ; do echo doSplit.csh `basename $file .maf` >> jobList ; done

    para create jobList
        # 43 jobs
    para try; para check
    para push

    # now set up exoniphy run

    mkdir -p /cluster/data/hg17/bed/exoniphy/test/run-exoniphy
    cd /cluster/data/hg17/bed/exoniphy/test/run-exoniphy
    cp -p ../../train/indels.hmm /cluster/bluearc/hg17/exoniphy/training
    mkdir -p /cluster/bluearc/hg17/exoniphy/GFF
    
    cat > doExoniphy.sh << 'EOF'
#!/usr/local/bin/bash

root=`basename $1 .ss.gz`
chrom=`echo $root | awk -F\. '{print $1}'`
no=`echo $root | awk 'BEGIN{FS="[-.]"} {printf "%d\n", ($2+10000)/100000}'`

if [ ! -d /cluster/bluearc/hg17/exoniphy/GFF/$chrom ] ; then 
    mkdir -p /cluster/bluearc/hg17/exoniphy/GFF/$chrom
fi

zcat $1 | /cluster/bin/phast/exoniphy - --hmm  /cluster/bluearc/hg17/exoniphy/training/indels.hmm --reflect-strand --extrapolate default --score --indels --alias "hg17=human; mm7=mouse; rn3=rat; canFam2=dog" --seqname $chrom --idpref $chrom.$no  > /cluster/bluearc/hg17/exoniphy/GFF/$chrom/$root.gff
'EOF'
# << for emacs
    chmod +x doExoniphy.sh

    rm -f jobList
    for dir in /cluster/bluearc/hg17/multiz.hg17Mm7Rn3CanFam2/ssSynFrags/* ; do find $dir -name '*.ss.gz' | awk '{printf "doExoniphy.sh %s\n", $1}' >> jobList ; done

    para create jobList
        # 27070 jobs
    para try; para check
    para push

#Completed: 27059 of 27070 jobs
#Crashed: 11 jobs
#CPU time in finished jobs:    8573545s  142892.41m  2381.54h   99.23d  0.272 y
#IO & Wait Time:                 73412s    1223.54m    20.39h    0.85d  0.002 y
#Average job time:                 320s       5.33m     0.09h    0.00d
#Longest running job:                0s       0.00m     0.00h    0.00d
#Longest finished job:             593s       9.88m     0.16h    0.01d
#Submission to last job:         22823s     380.38m     6.34h    0.26d    

    # crashed jobs all on random chroms, chrM, etc., and appear to be
    # due to all species not being present; okay to ignore

    # collect predictions and create track
    rm -f exoniphy.gff
    for dir in /cluster/bluearc/hg17/exoniphy/GFF/chr* ; do \
	rm -f files.* tmp.gff ;\
	find $dir -name "chr*.gff" > files ;\
	split -l 1000 files files. ;\
	for l in files.* ; do cat `cat $l` >> tmp.gff ; done ;\
	refeature --sort tmp.gff >> exoniphy.gff ;\
    done
    ldHgGene -genePredExt -gtf hg17 exoniphyDog exoniphy.gff


# COW SYNTENY (Done, Heather, Dec. 2005)
# Data from Harris A. Lewin <h-lewin@uiuc.edu>

ssh hgwdev
cd /cluster/data/hg17/bed
mkdir syntenyCow
cd syntenyCow

hgLoadBed -noBin hg17 syntenyCow syntenyCow.bed

# add to kent/src/hg/makeDb/trackDb/human/hg17/trackDb.ra
 
###########################################################################
# New Conservation track (WORKING 2005-12-15 kate)

# Pairwise alignments needed for: monDom2, danRer3, bosTau2
# Use existing alignments for:  
# macaque_rheMac1
# rat_rn3 
# mouse_mm7
# dog_canFam2
# chicken_galGal2
# xenopus_xenTro1
# fugu_fr1
# rabbit_oryCun1
# armadillo_dasNov1
# elephant_loxAfr1
# tenrec_echTel1
# tetraodon_tetNig1

#########################################################################
# BLASTZ danRer3 (DONE - 2005-12-20 kate)
# Includes both randoms
    ssh pk
    mkdir /cluster/data/hg17/bed/blastz.danRer3.2005-12-20
    cd /cluster/data/hg17/bed
    ln -s blastz.danRer3.2005-12-20 blastz.danRer3
    cd blastz.danRer3

    cat << 'EOF' > DEF
# human target, zebrafish query
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/x86_64:/cluster/home/angie/schwartzbin:/parasol/bin

BLASTZ=blastz.v7.x86_64
BLASTZ_ABRIDGE_REPEATS=1

# use parameters suggested for human-fish evolutionary distance
# recommended in doBlastzChainNet.pl help
# (previously used for  hg16-fr1, danrer1-mm5)
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=6000
BLASTZ_K=2200
BLASTZ_Q=/san/sanvol1/scratch/blastz/HoxD55.q

# TARGET: Human hg17
SEQ1_DIR=/san/sanvol1/scratch/hg17/nib
SEQ1_SMSK=/cluster/bluearc/hg17/linSpecRep.notInZebrafish
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_LEN=/san/sanvol1/scratch/hg17/chrom.sizes

# QUERY: zebrafish danRer3
# Use all chroms, including both randoms (chrUn and chrNA)
SEQ2_DIR=/san/sanvol1/scratch/danRer3/nib
SEQ2_SMSK=/san/sanvol1/scratch/danRer3/linSpecRep.notInOthers
SEQ2_LEN=/cluster/bluearc/danRer3/chrom.sizes
SEQ2_CHUNK=30000000
SEQ2_LAP=1000

BASE=/cluster/data/hg17/bed/blastz.danRer3.2005-12-20
TMPDIR=/scratch/tmp
'EOF'
    # << happy emacs

    /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk -chainMinScore=5000 -chainLinearGap=loose  \
        -stop=net \
	`pwd`/DEF >& blastz.out &

    # mistakenly started this in blastz.danRer3.2005-12-18 dir --
    # need to move DEF file and blastz.out to 2005-12-20 dir.

    # bogus stop at net step -- thinks it can't find chains
    # I'm just restarting there
    /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk -chainMinScore=5000 -chainLinearGap=loose  \
        -continue=net  \
	`pwd`/DEF >& blastz.2.out &

    # stopped because vsDanRer3 downloads already there from
    # previous run. 
    ssh hgwdev "rm -fr /usr/local/apache/htdocs/goldenPath/hg17/vsDanRer3"
    /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk -chainMinScore=5000 -chainLinearGap=loose  \
        -continue=download  \
	`pwd`/DEF >& blastz.3.out &

    # measurements
    ssh hgwdev "featureBits hg17 chainDanRer2Link" >& danRer2.fb; cat danRer2.fb
        # 70696998 bases of 2866216770 (2.467%) in intersection
    ssh hgwdev "featureBits hg17 chainDanRer3Link" >& danRer3.fb; cat danRer3.fb
        # 55625762 bases of 2866216770 (1.941%) in intersection

    # not sure why there's lower coverage from the newer assembly.
    # It's possibly due to different parameters used in the other
    # alignment.  Rachel is experimenting with hg18/danRer3, and
    # if warranted, we might replace this later

#########################################################################
# BLASTZ bosTau2 (DONE - 2005-12-19 kate)
    ssh pk
    mkdir /cluster/data/hg17/bed/blastz.bosTau2.2005-12-19
    cd /cluster/data/hg17/bed
    rm blastz.bosTau2
    ln -s blastz.bosTau2.2005-12-19 blastz.bosTau2
    cd blastz.bosTau2

    cat << 'EOF' > DEF
# human vs. cow
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn/x86_64:/cluster/bin/x86_64:/parasol/bin

BLASTZ=blastz.x86_64

# using parameter used when not using lineage specific repeat
# abridging.  This parameter restricts the # matches used by
# dynamic masking.  (We can't currently use LSR repeat abridging
# when either assembly sequence is in .2bit).

BLASTZ_M=50

# TARGET: Human (hg17)
SEQ1_DIR=/san/sanvol1/scratch/hg17/nib
SEQ1_LEN=/san/sanvol1/scratch/hg17/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Cow (bosTau2)
# chunk it as we can't do whole-genome on 2bits
SEQ2_DIR=/scratch/hg/bosTau2/bosTau2.noBin0.2bit
SEQ2_LEN=/scratch/hg/bosTau2/noBin0.sizes
SEQ2_CHUNK=300000000
SEQ2_LAP=10000

BASE=/cluster/data/hg17/bed/blastz.bosTau2.2005-12-19
TMPDIR=/scratch/tmp
'EOF'
    # << happy emacs
    # use chain parameters for "close" species
    /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk -chainMinScore=3000 -chainLinearGap=medium \
	`pwd`/DEF >& blastz.out &

    ssh hgwdev "featureBits hg17 chainBosTau1Link" >& bosTau1.fb; cat bosTau1.fb
    ssh hgwdev "featureBits hg17 chainBosTau2Link" >& bosTau2.fb; cat bosTau2.fb

    #	swapping to get the lift over file in the other direction (Hiram)
    ssh pk
    mkdir /cluster/data/bosTau2/bed/blastz.hg17.swap
    cd /cluster/data/bosTau2/bed
    ln -s blastz.hg17.swap blastz.hg17
    cd blastz.hg17.swap
    
    time /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-swap -bigClusterHub=pk -chainMinScore=3000 -chainLinearGap=medium \
	/cluster/data/hg17/bed/blastz.bosTau2.2005-12-19/DEF > swap.out 2>&1 &
    #	this failed during the load of the tables, but that is OK, we
    #	just wanted the liftOver files from this

    #	manually cleaned this up since the run faild during the MySQL
    #	load due to out of space problems.  These tables do not need to
    #	be loaded anyway.
    sh kkstore02
    cd /cluster/data/bosTau2/bed/blastz.hg17.swap

    rm -fr psl/

    rm -fr axtChain/run/chain/
    rm -f  axtChain/noClass.net
    rm -fr axtChain/net/
    rm -fr axtChain/chain/

#########################################################################
# BLASTZ rheMac2 (2006-02-08 kate)
    ssh pk
    mkdir /cluster/data/hg17/bed/blastz.rheMac2.2006-02-08
    cd /cluster/data/hg17/bed
    ln -s blastz.rheMac2.2006-02-08 blastz.rheMac2
    cd blastz.rheMac2

    cat << 'EOF' > DEF
# macaca mulatta vs. hg18
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn/x86_64:/cluster/bin/x86_64:/parasol/bin

ALIGN=blastz-run
BLASTZ=blastz.v7.x86_64

# TARGET - hg17
SEQ1_DIR=/san/sanvol1/scratch/hg17/nib
SEQ1_CHUNK=10000000
SEQ1_LAP=10000
SEQ1_LEN=/san/sanvol1/scratch/hg17/chrom.sizes

# QUERY - macaca mulatta
SEQ2_DIR=/san/sanvol1/scratch/rheMac2/rheMac2.2bit
SEQ2_CHUNK=5000000
SEQ2_LAP=0
SEQ2_LEN=/san/sanvol1/scratch/rheMac2/rheMac2.sizes 

BASE=/san/sanvol1/scratch/hg17/blastz.rheMac2/
RAW=$BASE/raw
TMPDIR=/scratch/tmp
'EOF'
    # << happy emacs
    # use chain parameters for "close" species
    /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk -chainMinScore=3000 -chainLinearGap=medium \
	`pwd`/DEF >& blastz.out &

    /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk -chainMinScore=3000 -chainLinearGap=medium \
	-continue=chainRun \
	`pwd`/DEF >& continueChainRun.out &

    # NOTE: must set -fileServer (e.g. to pk) if using base dir on SAN

    /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk -fileServer=pk \
        -chainMinScore=3000 -chainLinearGap=medium \
	-continue=chainMerge \
	`pwd`/DEF >& continueChainMerge.out &

    # netClass was crashing as it expected a bin in the
    # unsplit gap table.  Robert added the bin field.

    /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk -fileServer=pk \
	-continue=download \
	`pwd`/DEF >& continueDownload.out &

    ssh hgwdev "featureBits hg17 chainRheMac1Link" >& rheMac1.fb; cat rheMac1.fb
    ssh hgwdev "featureBits hg17 chainRheMac2Link" >& rheMac2.fb; cat rheMac2.fb

    ssh kkstore02 
    cd /cluster/data/hg17/bed/blastz.rheMac2
    cp -rp mafNet /san/sanvol1/scratch/hg17/mafNet/rheMac2

# SWAP CHAIN AND NET ALIGNMENTS OVER TO RHESUS (rheMac2) 
# CREATE CHAIN AND NET TRACKS, AXTNET, MAFNET, LIFTOVER AND ALIGNMENT DOWNLOADS
# (DONE, 2006-03-22, hartera) 
    # Do the swap of hg17/rheMac2 alignments over to rheMac2 to produce
    # rheMac2/hg17 alignments. 
    ssh pk
    cd /cluster/data/hg17/bed/blastz.rheMac2
    # use chain parameters for "close" species
    /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
        -swap -bigClusterHub=pk -chainMinScore=3000 -chainLinearGap=medium \
        `pwd`/DEF >& swap.log &
    # Took about 3 hours 40 minutes to run.

#############################################################################
# 17-WAY MULTIZ ALIGNMENTS (DONE - 2005-12-20 kate)
#       # redo fix overlaps from xenTro1 and tetNig1 (2006-04-08 kate)

    # copy net mafs to cluster-friendly storage for multiz run (2006-01-25 kate)

    ssh kkstore01
    cd /cluster/data/hg17/bed/blastz.monDom2
    cp -rp mafNet /san/sanvol1/scratch/hg17/mafNet/monDom2

    ssh kkstore02
    cd /cluster/data/hg17/bed
    mkdir -p multiz17way.2005-12-20
    ln -s multiz17way.2005-12-20 multiz17way
    cd multiz17way

    # copy MAF's to cluster-friendly server
    # These MAF's already on bluearc:
    #  canFam2, fr1, galGal2, panTro1, rn3
    mkdir -p /san/sanvol1/scratch/hg17/mafNet
    cd /san/sanvol1/scratch/hg17/mafNet
    ln -s /cluster/bluearc/hg17/mafNet/{*} .

    # copy others
    foreach s (rheMac1 oryCun1 dasNov1 \
            loxAfr1 bosTau2 monDom1 xenTro1 tetNig1 danRer3)
        echo $s
        cp -Rp /cluster/data/hg17/bed/blastz.$s/mafNet $s
    end
    # a few more
    set s = echTel1
    cp -Rp /cluster/data/hg17/bed/blastz.$s/mafNet $s
    set s = mm7
    cp -Rp /cluster/data/hg17/bed/blastz.$s/mafNet $s
    set s = canFam2
    cp -Rp /cluster/data/hg17/bed/blastz.$s/mafNet $s
    set s = rheMac2
    cp -Rp /cluster/data/hg17/bed/blastz.$s/mafNet $s

    # thanks for the tree, Hiram! Taken from mm7 17way...
    # Hiram says this is derived from the latest ENCODE
    # tree, with some species removed and branch lengths
    # adjusted.  The ENCODE tree  from the Sept. freeze is:
    # ftp://kronos.nhgri.nih.gov/pub/outgoing/elliott/msa/SEP-2005/phylo/tree_4d.tba.v2.nh
    cd /cluster/data/hg17/bed/multiz17way
    cat << '_EOF_' > 17way.nh
(((((((((
(human_hg17:0.006690,chimp_panTro1:0.007571):0.024272,
  macaque_rheMac2:0.0592):0.023960,
  ((rat_rn3:0.081728,mouse_mm7:0.077017):0.229273,
      rabbit_oryCun1:0.206767):0.1065):0.023026,
(cow_bosTau2:0.159182,dog_canFam2:0.147731):0.039450):0.028505,
armadillo_dasNov1:0.149862):0.015994,
(elephant_loxAfr1:0.104891,tenrec_echTel1:0.259797):0.040371):0.218400,
monodelphis_monDom2:0.371073):0.189124,
chicken_galGal2:0.454691):0.123297,
xenopus_xenTro1:0.782453):0.156067,
((tetraodon_tetNig1:0.199381,fugu_fr1:0.239894):0.492961,
    zebrafish_danRer3:0.782561):0.156067);
'_EOF_'

    /cluster/bin/phast/draw_tree 17way.nh > 17way.ps
    /cluster/bin/phast/all_dists 17way.nh > 17way.distances.txt
    grep hg17 17way.distances.txt | sort -k3,3n | \
        awk '{printf ("%.4f\t%s\n", $3, $2)}' > distances.txt
    # edit distances.txt to include featureBits, and chain parameters
    # from blastz run.
    cat distances.txt
        # 0.0143  chimp_panTro1
        # 0.0902  macaque_rheMac2
        # 0.2563  armadillo_dasNov1
        # 0.2651  dog_canFam2
        # 0.2677  elephant_loxAfr1
        # 0.2766  cow_bosTau2
        # 0.3682  rabbit_oryCun1
        # 0.4226  tenrec_echTel1
        # 0.4677  mouse_mm7
        # 0.4724  rat_rn3
    # use loose chain params and score from here, down (5000)
        # 0.7119  monodelphis_monDom1
        # 0.9847  chicken_galGal2
        # 1.4357  xenopus_xenTro1
        # 1.6577  tetraodon_tetNig1
        # 1.6983  fugu_fr1
        # 1.7480  zebrafish_danRer3

    # the order in the browser display will be by tree topology,
    #  not by distance, so it will be: 
#  >>         # 0.0143  chimp_panTro1
#  >>         # 0.0902  macaque_rheMac2
#  >>         # 0.4677  mouse_mm7
#  >>         # 0.4724  rat_rn3
#  >>         # 0.3682  rabbit_oryCun1
#  >>         # 0.2651  dog_canFam2
#  >>         # 0.2766  cow_bosTau2
#  >>         # 0.2563  armadillo_dasNov1
#  >>         # 0.2677  elephant_loxAfr1
#  >>         # 0.4226  tenrec_echTel1
#  >>         # 0.7119  monodelphis_monDom1
#  >>         # 0.9847  chicken_galGal2
#  >>         # 1.4357  xenopus_xenTro1
#  >>         # 1.6577  tetraodon_tetNig1
#  >>         # 1.6983  fugu_fr1
#  >>         # 1.7480  zebrafish_danRer3

    # make output dir and run dir
    ssh pk
    cd /cluster/data/hg17/bed/multiz17way.2005-12-20

    # create species list and stripped down tree for autoMZ
    sed 's/[a-z][a-z]*_//g; s/:[0-9\.][0-9\.]*//g; s/;//' 17way.nh > tmp.nh
    echo `cat tmp.nh` > tree-commas.nh
    echo `cat tree-commas.nh` | sed 's/ //g; s/,/ /g' > tree.nh
    sed 's/[()]//g; s/,/ /g' tree.nh > species.lst

    mkdir -p maf run
    cd run

    # stash binaries 
    mkdir penn
    cp -p /cluster/bin/penn/multiz.v11.x86_64/multiz-tba/multiz penn
    cp -p /cluster/bin/penn/multiz.v11.x86_64/multiz-tba/maf_project penn
    cp -p /cluster/bin/penn/multiz.v11.x86_64/multiz-tba/autoMZ penn

cat > autoMultiz.csh << 'EOF'
#!/bin/csh -ef
    set db = hg17
    set c = $1
    set maf = $2
    set run = `pwd`
    set tmp = /scratch/tmp/$db/multiz.$c
    set pairs = /san/sanvol1/scratch/$db/mafNet
    rm -fr $tmp
    mkdir -p $tmp
    cp ../{tree.nh,species.lst} $tmp
    pushd $tmp
    foreach s (`cat species.lst`)
        set in = $pairs/$s/$c.maf
        set out = $db.$s.sing.maf
        if ($s == hg17) then
            continue
        endif
        if (-e $in.gz) then
            zcat $in.gz > $out
        else if (-e $in) then
            cp $in $out
        else
            echo "##maf version=1 scoring=autoMZ" > $out
        endif
    end
    set path = ($run/penn $path); rehash
    $run/penn/autoMZ + T=$tmp E=$db "`cat tree.nh`" $db.*.sing.maf $c.maf
    popd
    cp $tmp/$c.maf $maf
    rm -fr $tmp
'EOF'
# << happy emacs
    chmod +x autoMultiz.csh

cat  << 'EOF' > spec
#LOOP
./autoMultiz.csh $(root1) {check out line+ /cluster/data/hg17/bed/multiz17way.2005-12-20/maf/$(root1).maf}
#ENDLOOP
'EOF'
# << happy emacs

    awk '{print $1}' /cluster/data/hg17/chrom.sizes > chrom.lst

    # REDO FOR OVERLAPS (2006-04-07 kate)
    mv ../maf ../maf.old
    # edit spec file to fix maf dir path
    gensub2 chrom.lst single spec jobList
    para create jobList
        # 46 files
    para try
    para check
    para push
    para time > run.time
        # 36 hrs (not typical -- previous runs were ~16 hrs)


# PHASTCONS CONSERVATION (2006-01-05 kate)
# Redone when multiz redone to fix overlaps (2006-04-12)
# This process is distilled from Hiram and Adam's experiments
# on mouse (mm7) 17way track.  Many parameters are now fixed, without
# being experimentally derived, either because the experiments
# were lengthy and produced similar results, or because they
# weren't runnable given the alignment size.
# These parameters are:
# --rho
# --expected-length
# --target-coverage
# Also, instead of generating cons and noncons tree models,
# we use a single, pre-existing tree model -- Elliot Margulies' model 
# from the (37-way) ENCODE alignments.
#
# NOTE: Redone 3/20/06, adding rheMac2 to non-informative options,
#       by recommendation of Adam Siepel, to correct unwanted
#       high conservation in regions with primate-only alignments

    # NOTE: reusing cluster-friendly chrom fasta files created earlier
    #cd /cluster/data/hg17
    #foreach f (`cat chrom.lst`)
        #echo $f
        #cp $f/*.fa /cluster/bluearc/hg17/chrom
    #end

    # Split chromosome MAF's into windows and use to generate
    # "sufficient statistics" (ss) files for phastCons input
    # NOTE: as the SAN fs has lotsa space, we're leaving these
    # big (temp) files unzipped, to save time during phastCons run.
    # Note also the larger chunk sizes from previous runs -- this
    # reduces run-time on the split, slows down the actual phastCons
    # enough so jobs don't crash (jobs are very quick, just a minute
    # or so), and according to Adam, will produce better results.
    # The previous small chunks were probably required by
    # the phyloFit step, which we are no longer using for the
    # human alignments.
    ssh pk
    mkdir /cluster/data/hg17/bed/multiz17way.2005-12-20/cons
    cd /cluster/data/hg17/bed/multiz17way.2005-12-20/cons
    cp /san/sanvol1/scratch/mm7/cons/elliotsEncode.mod .
        # edit, changing rheMac1 -> rheMac2
    mkdir run.split
    cd run.split
    set WINDOWS = /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/ss
    rm -fr $WINDOWS
    mkdir -p $WINDOWS

    cat << 'EOF' > doSplit.csh
#!/bin/csh -ef
    set MAFS = /cluster/data/hg17/bed/multiz17way.2005-12-20/maf
    set WINDOWS = /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/ss
    cd $WINDOWS
    set c = $1
    echo $c
    rm -fr $c
    mkdir $c
    /cluster/bin/phast/$MACHTYPE/msa_split $MAFS/$c.maf -i MAF \
        -M /cluster/bluearc/hg17/chrom/$c.fa \
        -o SS -r $c/$c -w 10000000,0 -I 1000 -B 5000
    echo "Done" >> $c.done
'EOF'
# << happy emacs
    chmod +x doSplit.csh

    rm -f jobList
    foreach f (../../maf/*.maf) 
        set c = $f:t:r
	echo "doSplit.csh $c {check out line+ $WINDOWS/$c.done}" >> jobList
    end
    
    para create jobList
        # 46 jobs
    para try
    para check
    para push
# CPU time in finished jobs:       9511s     158.52m     2.64h    0.11d  0.000 y
# IO & Wait Time:                  5391s      89.85m     1.50h    0.06d  0.000 y
# Average job time:                 324s       5.40m     0.09h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:            2354s      39.23m     0.65h    0.03d
# Submission to last job:          2358s      39.30m     0.66h    0.03d

    # check tree model on 5MB chunk, using params recommended by Adam,
    # (to verify branch lengths on 2X species)
    # he ok'ed the results -- not necessary for next human run
    ssh kolossus
    cd /cluster/data/hg17/bed/multiz17way.2005-12-20/cons
    /cluster/bin/phast/$MACHTYPE/phyloFit -i SS -E -p MED -s HKY85 \
        --tree "`cat ../tree-commas.nh`" \
        /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/ss/chr7/chr7.115000658-120000000.ss \
        -o phyloFit.tree

    # Run phastCons
    #	This job is I/O intensive in its output files, thus it is all
    #	working over in /scratch/tmp/
    cd ..
    mkdir run.cons
    cd run.cons
    cat > doPhast.csh << 'EOF'
#!/bin/csh -fe
set c = $1
set f = $2
set len = $3
set cov = $4
set rho = $5
set tmp = /scratch/tmp/$f
mkdir -p $tmp
set san = /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
cp -p $san/ss/$c/$f.ss ../elliotsEncode.mod $tmp
pushd $tmp > /dev/null
/cluster/bin/phast/$MACHTYPE/phastCons $f.ss elliotsEncode.mod \
   --rho $rho --expected-length $len --target-coverage $cov --quiet \
    --not-informative panTro1,rheMac2 \
	--seqname $c --idpref $c --viterbi $f.bed --score > $f.pp
popd > /dev/null
mkdir -p $san/pp/$c $san/bed/$c
sleep 1
mv $tmp/$f.pp $san/pp/$c
mv $tmp/$f.bed $san/bed/$c
rm -fr $tmp
'EOF'
    # emacs happy
    chmod a+x doPhast.csh

    #	root1 == chrom name, file1 == ss file name without .ss suffix
    # Create gsub file
    cat > template << 'EOF'
#LOOP
doPhast.csh $(root1) $(file1) 14 .008 .28
#ENDLOOP
'EOF'
    #	happy emacs

    # Create parasol batch and run it
    pushd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
    ls -1 ss/chr*/chr*.ss | sed 's/.ss$//' > \
        /cluster/data/hg17/bed/multiz17way/cons/run.cons/in.list
    popd

    gensub2 in.list single template jobList
    para create jobList
        # 333 jobs
    para try
    para check
    para push
    # NOTE: these jobs go fast -- some crashed apparently having
    # difficulty accessing input files.  Just restart them and
    # they work
#CPU time in finished jobs:      15520s     258.67m     4.31h    0.18d  0.000 y
#IO & Wait Time:                 15796s     263.27m     4.39h    0.18d  0.001 y
#Average job time:                  94s       1.57m     0.03h    0.00d
#Longest running job:                0s       0.00m     0.00h    0.00d
#Longest finished job:             180s       3.00m     0.05h    0.00d
#Submission to last job:         48266s     804.43m    13.41h    0.56d

    # create Most Conserved track
    ssh kolossus
    cd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
    #	The sed's and the sort get the file names in chrom,start order
    # (Hiram tricks -- split into columns on [.-/] with 
    #    identifying x,y,z, to allow column sorting and
    #    restoring the filename.  Warning: the sort column
    # will depend on how deep you are in the dir
    find ./bed -name "chr*.bed" | \
        sed -e "s/\// x /g" -e "s/\./ y /g" -e "s/-/ z /g" | \
	sort -k7,7 -k9,9n | \
	sed -e "s/ x /\//g" -e "s/ y /\./g" -e "s/ z /-/g" | \
	xargs cat | \
	awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", $1, $2, $3, $5, $5;}' | \
	/cluster/bin/scripts/lodToBedScore /dev/stdin > mostConserved.bed
    #	~ 1 minute
    cp -p mostConserved.bed /cluster/data/hg17/bed/multiz17way/cons

    # load into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz17way/cons
    hgLoadBed -strict hg17 phastConsElements17way mostConserved.bed
        # Loaded 2212445 elements of size 5
    # compare with previous tracks
    hgsql hg17 -e "select count(*) from phastConsElements10way"
        # 2011952
    hgsql hg17 -e "select count(*) from phastConsElements"
        # 1601903
    # Try for 5% overall cov, and 70% CDS cov (used elen=14, tcov=.008, rho=.28)
    
    featureBits hg17 -enrichment refGene:cds phastConsElements17way
        # refGene:cds 1.065%, phastConsElements17way 5.116%, both 0.759%, cover 71.27%, enrich 13.93x

    # compare with previous tracks
    featureBits hg17 -enrichment refGene:cds phastConsElements10way
        # refGene:cds 1.062%, phastConsElements10way 5.003%, both 0.734%, cover 69.18%, enrich 13.83x
    featureBits hg17 -enrichment refGene:cds phastConsElements
        # refGene:cds 1.062%, phastConsElements 4.810%, both 0.771%, cover 72.65%, enrich 15.11x

    # experiments
    # previous tracks
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements
        # refGene:cds 0.873%, phastConsElements 4.497%, both 0.630%, cover 72.10%, enrich 16.04x
    hgsql hg17 -e "select count(*) from phastConsElements where chrom='chr7'"
        # 81785
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements10way
        # refGene:cds 0.873%, phastConsElements10way 4.700%, both 0.602%, cover 68.94%, enrich 14.67x
    hgsql hg17 -e "select count(*) from phastConsElements10way where chrom='chr7'"
        # 102959

    # len=13, cov=.007, rho=.27
    # looks best -- similar chr7 measurements to previous tracks
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_13_007_27
        # refGene:cds 0.874%, phastConsElements17way_13_007_27 4.854%, both 0.607%, cover 69.43%, enrich 14.31x
    hgsql hg17 -e "select count(*) from phastConsElements17way_13_007_27 where chrom='chr7'"

    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_13_005_28
       # refGene:cds 0.873%, phastConsElements17way_13_005_28 4.802%, both 0.612%, cover 70.12%, enrich 14.60x
    hgsql hg17 -e "select count(*) from phastConsElements17way_13_005_28 where chrom='chr7'"
       # 95203 

    # experiments with other parameters, below
    # len=15, cov=.10 
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_15_10
        # refGene:cds 0.873%, phastConsElements17way 7.989%, both 0.627%, cover 71.77%, enrich 8.98x
    hgsql hg17 -e "select count(*) from phastConsElements17way_15_10 where chrom='chr7'"
        # 217767 
    # => too much overall covg, and too many elements

    # len=15, cov=.05
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_15_05
        # refGene:cds 0.873%, phastConsElements17way_15_05 6.880%, both 0.627%, cover 71.77%, enrich 10.43x
    hgsql hg17 -e "select count(*) from phastConsElements17way_15_05 where chrom='chr7'"
        # 166868

    # len=15, cov=.01
    # These values were used by Elliott for ENCODE
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_15_01
        # refGene:cds 0.873%, phastConsElements17way_15_01 5.721%, both 0.628%, cover 71.89%, enrich 12.57x
    hgsql hg17 -e "select count(*) from phastConsElements17way_15_01 where chrom='chr7'"
        # 106034

    # len=20, cov=.01
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_20_01
        # refGene:cds 0.873%, phastConsElements17way_20_01 7.751%, both 0.634%, cover 72.56%, enrich 9.36x
    hgsql hg17 -e "select count(*) from phastConsElements17way_20_01 where chrom='chr7'"
        # 106005
    # -> wrong direction on coverage

    # len=10, cov=.01
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_10_01
        # refGene:cds 0.873%, phastConsElements17way_10_01 4.653%, both 0.616%, cover 70.48%, enrich 15.15x
    hgsql hg17 -e "select count(*) from phastConsElements17way_10_01 where chrom='chr7'"
        # 108279
    # => looks good on coverage and element count, check smoothness in browser
    # => undersmoothed

    # len=10, cov=.05
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_10_05
        # refGene:cds 0.873%, phastConsElements17way_10_05 5.365%, both 0.615%, cover 70.44%, enrich 13.13x 
    hgsql hg17 -e "select count(*) from phastConsElements17way_10_05 where chrom='chr7'"
        # 178372
    # => fragmented elements

    # len=15, cov=.005
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_15_005
        # refGene:cds 0.873%, phastConsElements17way_15_005 5.444%, both 0.628%, cover 71.93%, enrich 13.21x
    hgsql hg17 -e "select count(*) from phastConsElements17way_15_005 where chrom='chr7'"
        # 90855

    # len=20, cov=.005
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_20_005
        # refGene:cds 0.873%, phastConsElements17way_20_005 7.373%, both 0.634%, cover 72.61%, enrich 9.85x
    hgsql hg17 -e "select count(*) from phastConsElements17way_20_005 where chrom='chr7'"
        # 91858

    # len=17, cov=.005 rho=.3
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_17_005
        # refGene:cds 0.873%, phastConsElements17way_17_005 6.126%, both 0.631%, cover 72.24%, enrich 11.79x
    hgsql hg17 -e "select count(*) from phastConsElements17way_17_005 where chrom='chr7'"
        # 91243

    # len=12, cov=.01, rho=.28 -panTro1
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_12_01_28_p
        # refGene:cds 0.873%, phastConsElements17way_12_01_28_p 4.829%, both 0.612%, cover 70.02%, enrich 14.50x
    hgsql hg17 -e "select count(*) from phastConsElements17way_12_01_28_p where chrom='chr7'"
        # 123638

    # len=13, cov=.01, rho=.25 -panTro1
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_13_01_25_p
        # refGene:cds 0.873%, phastConsElements17way_13_01_25_p 4.793%, both 0.594%, cover 67.99%, enrich 14.19x
    hgsql hg17 -e "select count(*) from phastConsElements17way_13_01_25_p where chrom='chr7'"
        # 131895

    # len=14, cov=.008, rho=.28
    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way_14_008_28
       # refGene:cds 0.874%, phastConsElements17way_14_008_28 5.227%, both 0.615%, cover 70.37%, enrich 13.46x
    hgsql hg17 -e "select count(*) from phastConsElements17way_14_008_28 where chrom='chr7'"
        # 106071

    # Create merged posterier probability file and wiggle track data files
    #	pk is currently closer to the san than any other machine
    ssh pk
    cd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/
    # sort by chromName, chromStart so that items are in numerical order 
    #  for wigEncode
    #next time try Angie's simpler sort, below
    find ./pp -name "chr*.pp" | \
        sed -e "s/\// x /g" -e "s/\./ y /g" -e "s/-/ z /g" | \
	sort -k7,7 -k9,9n | \
	sed -e "s/ x /\//g" -e "s/ y /\./g" -e "s/ z /-/g" | \
	xargs cat | \
        nice wigEncode stdin phastCons17way.wig phastCons17way.wib
    # about 23 minutes for above
        # GOT HERE ON REDO
        # NOTE: remember to flip /gbdb link from cons.old to cons
    #foreach chr (`awk '{print $1}' /cluster/data/hg17/chrom.sizes`)
      #echo $chr
      set chr = chr22
      cat `ls -1 pp/$chr/$chr.*.pp | sort -t\. -k2,2n` \
        | nice wigEncode stdin phastCons17wayNewChr22.wig phastCons17wayNewChr22.wib
    #end
    date

    cp -p phastCons17way.wi? /cluster/data/hg17/bed/multiz17way/cons

    # Load gbdb and database with wiggle.
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz17way/cons
    ln -s /cluster/data/hg17/bed/multiz17way/cons/phastCons17way.wib \
        /gbdb/hg17/multiz17way/phastCons17way.wib
    hgLoadWiggle -pathPrefix=/gbdb/hg17/multiz17way hg17 \
        phastCons17way phastCons17way.wig

############################################################################
    ## Run phastCons on Placental mammals
    ssh pk
    cd /cluster/data/hg17/bed/multiz17way.2005-12-20/cons
    mkdir placental
    mkdir run.cons.alt
    cd run.cons.alt

    # create pruned trees 
    set tree_doctor = /cluster/bin/phast/tree_doctor
    sed 's/ /,/g' ../../species.lst 
    # hg17,panTro1,rheMac2,rn3,mm7,oryCun1,bosTau2,canFam2,dasNov1,loxAfr1,echTel1,monDom2,galGal2,xenTro1,tetNig1,fr1,danRer3
    mkdir placental
    $tree_doctor ../elliotsEncode.mod \
        --prune-all-but=hg17,panTro1,rheMac2,rn3,mm7,oryCun1,bosTau2,canFam2,dasNov1,loxAfr1,echTel1 \
                > placental/placental.mod

    cat > doPhast.csh << 'EOF'
#!/bin/csh -fe
set c = $1
set f = $2
set len = $3
set cov = $4
set rho = $5
set grp = $6
set tmp = /scratch/tmp/hg17/$grp/$f
mkdir -p $tmp
set san = /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
cp -p $san/ss/$c/$f.ss $grp/$grp.mod $tmp
pushd $tmp > /dev/null
/cluster/bin/phast/$MACHTYPE/phastCons $f.ss $grp.mod \
   --rho $rho --expected-length $len --target-coverage $cov --quiet \
    --not-informative panTro1,rheMac2 \
	--seqname $c --idpref $c --viterbi $f.bed --score > $f.pp
popd > /dev/null
mkdir -p $san/$grp/pp/$c $san/$grp/bed/$c
sleep 1
mv $tmp/$f.pp $san/$grp/pp/$c
mv $tmp/$f.bed $san/$grp/bed/$c
rm -fr $tmp
'EOF'
    # << emacs happy

    chmod a+x doPhast.csh

    # Create gsub file
    cat > template << 'EOF'
#LOOP
# template for 5% cov
doPhast.csh $(root1) $(file1) 14 .2 .28 placental
#ENDLOOP
'EOF'
    cat > template << 'EOF'
#LOOP
# template same as vertebrate
doPhast.csh $(root1) $(file1) 14 .008 .28 placental
#ENDLOOP
'EOF'
    #	happy emacs

    # Create parasol batch and run it
    pushd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
    ls -1 ss/chr*/chr*.ss | sed 's/.ss$//' > \
        /cluster/data/hg17/bed/multiz17way/cons/run.cons.alt/in.list
    popd

    gensub2 in.list single template jobList
    para create jobList
        # 333 jobs
    para try
    para check
    para push
#.2
#CPU time in finished jobs:      15164s     252.74m     4.21h    0.18d  0.000 y
#IO & Wait Time:                 14852s     247.53m     4.13h    0.17d  0.000 y
#Average job time:                  90s       1.50m     0.03h    0.00d
#Longest running job:                0s       0.00m     0.00h    0.00d
#Longest finished job:             170s       2.83m     0.05h    0.00d
#Submission to last job:         86364s    1439.40m    23.99h    1.00d

#.008
#CPU time in finished jobs:      13712s     228.53m     3.81h    0.16d  0.000 y
#IO & Wait Time:                 14407s     240.12m     4.00h    0.17d  0.000 y
#Average job time:                  84s       1.41m     0.02h    0.00d
#Longest running job:                0s       0.00m     0.00h    0.00d
#Longest finished job:             159s       2.65m     0.04h    0.00d
#Submission to last job:          5291s      88.18m     1.47h    0.06d

    ssh pk
    cd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/placental
    #	The sed's and the sort get the file names in chrom,start order
    # (Hiram tricks -- split into columns on [.-/] with 
    #    identifying x,y,z, to allow column sorting and
    #    restoring the filename.  Warning: the sort column
    # will depend on how deep you are in the dir
    find ./bed -name "chr*.bed" | \
        sed -e "s/\// x /g" -e "s/\./ y /g" -e "s/-/ z /g" | \
	sort -k7,7 -k9,9n | \
	sed -e "s/ x /\//g" -e "s/ y /\./g" -e "s/ z /-/g" | \
	xargs cat | \
	awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", $1, $2, $3, $5, $5;}' | \
	/cluster/bin/scripts/lodToBedScore /dev/stdin > mostConserved.bed
    #	~ 1 minute
    cp -p mostConserved.bed /cluster/data/hg17/bed/multiz17way/cons/placental

    # load into database
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz17way/cons/placental
    hgLoadBed -strict hg17 phastConsElementsPlacental mostConserved.bed
        # .2
        # Loaded 3775983 elements of size 5
        # .008
        # Loaded 1290060 elements of size 5

    # compare with vertebrate cons
    hgsql hg17 -e "select count(*) from phastConsElements17way"
        # 2212445
    featureBits hg17 -enrichment refGene:cds phastConsElementsPlacental
    featureBits hg17 -enrichment refGene:cds phastConsElementsPlacental_14_2_28
    featureBits hg17 -enrichment refGene:cds phastConsElements17way

    featureBits hg17 -enrichment refGene:cds phastConsElementsPlacental
# refGene:cds 1.070%, phastConsElementsPlacental 3.844%, both 0.667%, cover 62.32%, enrich 16.21x
# refGene:cds 1.069%, phastConsElementsPlacental_14_008_28 3.844%, both 0.667%, cover 62.37%, enrich 16.22x

    featureBits hg17 -enrichment refGene:cds phastConsElementsPlacental_14_2_28
#refGene:cds 1.070%, phastConsElementsPlacental_14_2_28 5.223%, both 0.691%, cover 64.62%, enrich 12.37x
    featureBits hg17 -enrichment refGene:cds phastConsElements17way
#refGene:cds 1.070%, phastConsElements17way 5.116%, both 0.763%, cover 71.27%, enrich 13.93x


    # Create merged posterier probability file and wiggle track data files
    #	pk is currently closer to the san than any other machine
    ssh pk
    cd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/placental
    # sort by chromName, chromStart so that items are in numerical order 
    #  for wigEncode
    #next time try Angie's simpler sort, below
    find ./pp -name "chr*.pp" | \
        sed -e "s/\// x /g" -e "s/\./ y /g" -e "s/-/ z /g" | \
	sort -k7,7 -k9,9n | \
	sed -e "s/ x /\//g" -e "s/ y /\./g" -e "s/ z /-/g" | \
	xargs cat | \
        nice wigEncode stdin phastConsPlacental.wig phastConsPlacental.wib
    # about 23 minutes for above
        # GOT HERE ON REDO
        # NOTE: remember to flip /gbdb link from cons.old to cons

    cp -p phastConsPlacental.wi?  \
                /cluster/data/hg17/bed/multiz17way/cons/placental

    # Load gbdb and database with wiggle.
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz17way/cons/placental
    ln -s \
     /cluster/data/hg17/bed/multiz17way/cons/placental/phastConsPlacental.wib \
        /gbdb/hg17/multiz17way/phastConsPlacental.wib
    hgLoadWiggle -pathPrefix=/gbdb/hg17/multiz17way hg17 \
        phastConsPlacental phastConsPlacental.wig

############################################################################
    ## Run phastCons on subgroups (mammals, placentals, and w/o low-cov)
    ssh pk
    cd /cluster/data/hg17/bed/multiz17way.2005-12-20/cons
    mkdir run.cons.groups
    cd run.cons.groups

    # create pruned trees 
    set tree_doctor = /cluster/bin/phast/tree_doctor
    sed 's/ /,/g' ../../species.lst 
    # hg17,panTro1,rheMac2,rn3,mm7,oryCun1,bosTau2,canFam2,dasNov1,loxAfr1,echTel1,monDom2,galGal2,xenTro1,tetNig1,fr1,danRer3

    $tree_doctor ../elliotsEncode.mod \
        --prune-all-but=hg17,panTro1,rheMac2,rn3,mm7,bosTau2,canFam2,monDom2,galGal2,xenTro1,tetNig1,fr1,danRer3 \
                > vertebrate-high.mod
    $tree_doctor ../elliotsEncode.mod \
        --prune-all-but=hg17,panTro1,rheMac2,rn3,mm7,bosTau2,canFam2,monDom2 \
                > mammal-high.mod
    $tree_doctor ../elliotsEncode.mod \
        --prune-all-but=hg17,panTro1,rheMac2,rn3,mm7,bosTau2,canFam2 \
                > placental-high.mod
    $tree_doctor ../elliotsEncode.mod \
        --prune-all-but=hg17,panTro1,rheMac2,rn3,mm7,oryCun1,bosTau2,canFam2,dasNov1,loxAfr1,echTel1,monDom2 \
                > mammal.mod
    $tree_doctor ../elliotsEncode.mod \
        --prune-all-but=hg17,panTro1,rheMac2,rn3,mm7,oryCun1,bosTau2,canFam2,dasNov1,loxAfr1,echTel1 \
                > placental.mod
    foreach f (*.mod)
        mkdir $f:r
        mv $f $f:r
    end
    cat > doPhast.csh << 'EOF'
#!/bin/csh -fe
set c = $1
set f = $2
set len = $3
set cov = $4
set rho = $5
set grp = $6
set tmp = /scratch/tmp/hg17/$grp/$f
mkdir -p $tmp
set san = /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
cp -p $san/ss/$c/$f.ss $grp/$grp.mod $tmp
pushd $tmp > /dev/null
/cluster/bin/phast/$MACHTYPE/phastCons $f.ss $grp.mod \
   --rho $rho --expected-length $len --target-coverage $cov --quiet \
    --not-informative panTro1,rheMac2 \
	--seqname $c --idpref $c --viterbi $f.bed --score > $f.pp
popd > /dev/null
mkdir -p $san/$grp/pp/$c $san/$grp/bed/$c
sleep 1
mv $tmp/$f.pp $san/$grp/pp/$c
mv $tmp/$f.bed $san/$grp/bed/$c
rm -fr $tmp
'EOF'
    # emacs happy
    chmod a+x doPhast.csh

    #	root1 == chrom name, file1 == ss file name without .ss suffix
    # Create gsub file
    cat > template << 'EOF'
#LOOP
doPhast.csh $(root1) $(file1) 14   .21 .28 placental-high
doPhast.csh $(root1) $(file1) 14   .2 .28 placental
doPhast.csh $(root1) $(file1) 14   .11 .28 mammal
doPhast.csh $(root1) $(file1) 14   .1 .28 mammal-high
doPhast.csh $(root1) $(file1) 14 .0028 .28 vertebrate-high
#ENDLOOP

'EOF'
    #	happy emacs

    # Create parasol batch for just chr7 (for test purposes) and run it
    pushd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
    ls -1 ss/chr7/chr*.ss | sed 's/.ss$//' > \
        /cluster/data/hg17/bed/multiz17way/cons/run.cons.groups/in.list
    popd

    gensub2 in.list single template jobList
    para create jobList
        # 80 jobs
    para try
    para check
    para push
    # 24 minutes


    ## create Alt Most Conserved track
    ssh hgwdev
    cd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
cat > loadAltElements.csh << 'EOF'
    set b = /cluster/data/hg17/bed/multiz17way/cons/run.cons.groups
    foreach d (mammal* placental* vertebrate*)
      echo $d
      cd $d
      find ./bed -name "chr*.bed" | \
        sed -e "s/\// x /g" -e "s/\./ y /g" -e "s/-/ z /g" | \
	sort -k7,7 -k9,9n | \
	sed -e "s/ x /\//g" -e "s/ y /\./g" -e "s/ z /-/g" | \
	xargs cat | \
	awk '{printf "%s\t%d\t%d\tlod=%d\t%s\n", $1, $2, $3, $5, $5;}' | \
	/cluster/bin/scripts/lodToBedScore /dev/stdin \
            > $b/$d/mostConserved.bed
        set table = `echo $d | perl -wpe "s/(.*)/phastConsElements\u$1/;s/-(.*)/\u$1/"`
        hgLoadBed -strict hg17 $table $b/$d/mostConserved.bed
        featureBits hg17 -enrichment refGene:cds -chrom=chr7 $table
      cd ..
    end
'EOF'
    csh loadAltElements.csh >&! loadAltElements.log &
    grep refGene loadAltElements.log | sort -n -k4
# refGene:cds 0.884%, phastConsElementsPlacentalHigh 4.828%, both 0.606%, cover 68.51%, enrich 14.19x
# refGene:cds 0.884%, phastConsElementsMammal 4.869%, both 0.580%, cover 65.62%, enrich 13.48x
# refGene:cds 0.884%, phastConsElementsMammalHigh 4.887%, both 0.624%, cover 70.60%, enrich 14.45x
# refGene:cds 0.884%, phastConsElementsPlacental 4.904%, both 0.558%, cover 63.14%, enrich 12.88x
# refGene:cds 0.884%, phastConsElementsVertebrateHigh 4.965%, both 0.652%, cover 73.74%, enrich 14.85x

    featureBits hg17 -enrichment refGene:cds -chrom=chr7 phastConsElements17way
# refGene:cds 0.884%, phastConsElements17way 4.851%, both 0.623%, cover 70.48%, enrich 14.53x

    ssh kkstore02
    cd /cluster/data/hg17/bed/multiz17way/cons/run.cons.groups
cat > makeAltWiggle.csh << 'EOF'
    set b = `pwd`
    set san = /san/sanvol1/scratch/hg17/multiz17way/cons
    pushd $san
    foreach d (mammal* placental* vertebrate*)
      echo $d
      cd $d
      set table = `echo $d | perl -wpe 's/(.*)/phastCons\u$1/;s/-(.*)/\u$1/'`
      echo $table
      find ./pp -name "chr*.pp" | \
        sed -e "s/\// x /g" -e "s/\./ y /g" -e "s/-/ z /g" | \
	sort -k7,7 -k9,9n | \
	sed -e "s/ x /\//g" -e "s/ y /\./g" -e "s/ z /-/g" | \
	xargs cat | \
        nice wigEncode stdin $table.wig $table.wib
      mv $table.wig $table.wib $b/$d
      cd ..
    end
    popd
'EOF'
    csh makeAltWiggle.csh >&! makeAltWiggle.log &

    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz17way/cons/run.cons.groups
cat > loadAltWiggle.csh << 'EOF'
    set b = `pwd`
    foreach d (mammal* placental* vertebrate*)
        echo $d
        cd $d
        set table = `echo $d | perl -wpe 's/(.*)/phastCons\u$1/;s/-(.*)/\u$1/'`
        echo $table
        ln -s `pwd`/$table.wib /gbdb/hg17/multiz17way
        hgLoadWiggle -pathPrefix=/gbdb/hg17/multiz17way hg17 $table $table.wig
        cd ..
    end
'EOF'
    csh loadAltWiggle.csh >&! loadAltWiggle.log &

    # Create parasol batch for just chr7 (for test purposes) and run it
    pushd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons
    ls -1 ss/chr7/chr*.ss | sed 's/.ss$//' > \
        /cluster/data/hg17/bed/multiz17way/cons/run.cons.groups/in.list
    popd

    gensub2 in.list single template jobList
    para create jobList
        # 80 jobs
    para try
    para check
    para push
    # 24 minutes

    # Downloads  (2006-02-22 kate)
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz17way
    mkdir mafDownloads
    cd mafDownloads
    # upstream mafs 
cat > mafFrags.csh << 'EOF'
    date
    foreach i (1000 2000 5000)
        echo "making upstream$i.maf"
        nice featureBits hg17 refGene:upstream:$i -fa=/dev/null -bed=up.bad
        awk -F '\t' '{printf("%s\t%s\t%s\t%s\t%s\t%s\n", $1, $2, $3, substr($4, 0, 9), 0, $5)}' up.bad > up.bed
        rm up.bad
        nice mafFrags hg17 multiz17way up.bed upstream$i.maf \
                -orgs=../species.lst
        nice gzip upstream$i.maf
        rm up.bed
    end
    date
'EOF'
    time csh mafFrags.csh >&! mafFrags.log &
        # ~1 hour

    ssh kkstore02
    cd cluster/data/hg17/bed/multiz17way/mafDownloads
cat > downloads.csh << 'EOF'
    date
    foreach f (../maf/chr*.maf)
	set c = $f:t:r
        echo $c
	nice gzip -c $f > $c.maf.gz
    end
    md5sum *.gz > md5sum.txt
    date
'EOF'
    time csh downloads.csh >&! downloads.log
        # ~2 hours
        # GOT HERE

    ssh hgwdev
    set dir = /usr/local/apache/htdocs/goldenPath/hg17/multiz17way
    mkdir $dir
    ln -s /cluster/data/hg17/bed/multiz17way/mafDownloads/{*.gz,md5sum.txt} $dir
    cp /usr/local/apache/htdocs/goldenPath/mm7/multiz17way/README.txt $dir
    # edit README


# PHASTCONS SCORES DOWNLOADABLES FOR 17WAY (2006-03-20 kate)

    ssh kkstore02
    cd /cluster/data/hg17/bed/multiz17way 
    mkdir phastConsDownloads
    cd phastConsDownloads
cat > downloads.csh << 'EOF'
    date
    cd /san/sanvol1/scratch/hg17/multiz17way.2005-12-20/cons/pp
    foreach chr (`awk '{print $1}' /cluster/data/hg17/chrom.sizes`)
      echo $chr
      cat `ls -1 $chr/$chr.*.pp | sort -t\. -k2,2n` \
        | nice gzip -c \
            > /cluster/data/hg17/bed/multiz17way/phastConsDownloads/$chr.gz
    end
    date
'EOF'
    csh downloads.csh >&! downloads.log &

        # ~20 minutes
    # << happy emacs
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz17way/phastConsDownloads
    md5sum *.gz > md5sum.txt
    set dir = /usr/local/apache/htdocs/goldenPath/hg17/phastCons17way
    mkdir $dir
    ln -s /cluster/data/hg17/bed/multiz17way/phastConsDownloads/{*.gz,md5sum.txt} $dir
    cp /usr/local/apache/htdocs/goldenPath/hg17/phastCons/README.txt $dir
    # edit 


# UPDATE MONKEY DOWNLOADS (2006-01-12 kate)
# EXTRACT AXT'S AND MAF'S FROM THE RheMac1 NET
# The chr1 was hugely oversized -- the other's were OK, but
# axt's were numbered oddly.
    ssh kkstore2
    cd /cluster/data/hg17/bed/blastz.rheMac1/axtChain
    gunzip -c  hg17.rheMac1.net.gz | netSplit stdin humanNet
    gunzip -c hg17.rheMac1.all.chain.gz | chainSplit chain stdin
    mkdir ../axtNet.new ../mafNet.new
cat > makeMaf.csh << 'EOF'
    foreach f (humanNet/chr*.net)
        set c = $f:t:r
        echo "axtNet on $c"
        netToAxt humanNet/$c.net chain/$c.chain /cluster/data/hg17/nib /cluster/data/rheMac1/rheMac1.2bit stdout | axtSort stdin ../axtNet.new/$c.axt
        axtToMaf ../axtNet.new/$c.axt \
            /cluster/data/hg17/chrom.sizes /cluster/data/rheMac1/chrom.sizes \
            ../mafNet.new/$c.maf -tPrefix=hg17. -qPrefix=rheMac1.
    end
    cp -rp ../mafNet.new /san/sanvol1/scratch/hg17/mafNet/rheMac1.new
'EOF'
    csh makeMaf.csh >&! makeMaf.log &
    tail -100f makeMaf.log
    pushd /san/sanvol1/scratch/hg17/mafNet
    rm -fr rheMac1
    mv rheMac1.new rheMac1
    popd

    rm -fr axtNet
    mv axtNet.new axtNet
    cd axtNet
    nice gzip *.axt
    md5sum *.gz > md5sum.txt
    # cleanup
    cd ..
    rm -fr chain humanNet

    ssh hgwdev
    ln -s /cluster/data/hg17/bed/blastz.rheMac1/axtNet \
         /usr/local/apache/htdocs/goldenPath/rheMac1/axtNet
    # Request push to downloads server


# UPDATE OPOSSUM DOWNLOADS (2006-01-17 kate)
# Fix overlaps

    ssh kkstore2
    cd /cluster/data/hg17/bed/blastz.monDom1
    mv axtNet axtNet.old
    mv mafNet mafNet.old
    mkdir axtNet mafNet
    cd axtChain/chain
    nice gunzip *.gz
    cd ..
    nice gunzip -c  human.net.gz | netSplit stdin humanNet
cat > makeMaf.csh << 'EOF'
    foreach f (humanNet/chr*.net)
        set c = $f:t:r
        echo "axtNet on $c"
        netToAxt humanNet/$c.net chain/$c.chain /cluster/data/hg17/nib /cluster/data/monDom1/monDom1.2bit stdout | axtSort stdin ../axtNet/$c.axt
        axtToMaf ../axtNet/$c.axt \
            /cluster/data/hg17/chrom.sizes /cluster/data/monDom1/chrom.sizes \
            ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=monDom1.
    end
    cp -rp ../mafNet /san/sanvol1/scratch/hg17/mafNet/monDom1.new
'EOF'
    csh makeMaf.csh >&! makeMaf.log &
    tail -100f makeMaf.log
    pushd /san/sanvol1/scratch/hg17/mafNet
    rm -fr monDom1
    mv monDom1.new monDom1
    popd

    rm -fr axtNet
    mv axtNet.new axtNet
    cd axtNet
    nice gzip *.axt
    md5sum *.gz > md5sum.txt

    # cleanup
    cd ..
    rm -fr chain humanNet

    ssh hgwdev
    ln -s /cluster/data/hg17/bed/blastz.monDom1/axtNet \
         /usr/local/apache/htdocs/goldenPath/monDom1/axtNet
    # Request push to downloads server


# UPDATE COW DOWNLOADS (2006-01-17 kate)
# Fix overlaps

    ssh kkstore2
    cd /cluster/data/bosTau1/bed/zb.hg17
    mv axtNet axtNet.old
    mv mafNet mafNet.old
    mkdir axtNet mafNet
cat > makeMaf.csh << 'EOF'
    foreach f (net/chr*.net)
        set c = $f:t:r
        echo "axtNet on $c"
        netToAxt net/$c.net chain/$c.chain /cluster/data/hg17/nib /cluster/data/bosTau1/bosTau1.2bit stdout | axtSort stdin ../axtNet/$c.axt
        axtToMaf ../axtNet/$c.axt \
            /cluster/data/hg17/chrom.sizes /cluster/data/bosTau1/chrom.sizes \
            ../mafNet/$c.maf -tPrefix=hg17. -qPrefix=bosTau1.
    end
'EOF'
    csh makeMaf.csh >&! makeMaf.log &
    tail -100f makeMaf.log

    cd axtNet
    nice gzip *.axt
    md5sum *.gz > md5sum.txt

    ssh hgwdev
    ln -s /cluster/data/hg17/bed/blastz.bosTau1/axtNet \
         /usr/local/apache/htdocs/goldenPath/bosTau1/axtNet
    # Request push to downloads server


##### UPDATE hg17 knownToVisiGene (2006-01-21 galt)
# Create table that maps between known genes and visiGene database 
    # mapping to other species such as mouse, zebrafish, frog
    # requires visiGene probe track vgImageProbes be created first
    knownToVisiGene hg17 -fromProbePsl=vgImageProbes


##### UPDATE hg17 mmBlastTab (2006-01-22 galt)
# Make the protein seqs from mm7.knownGenePep
cd /cluster/data/hg17/bed/geneSorter/blastp
mkdir mm7
cd mm7
pepPredToFa mm7 knownGenePep known.faa
#       You may need to build this binary in src/hg/near/pepPredToFa
/cluster/bluearc/blast229/formatdb -i known.faa -t known -n known
mkdir -p /cluster/panasas/home/store/mm7/blastp/
cp known.* /cluster/panasas/home/store/mm7/blastp/

# Make parasol run directory
ssh kk
cd /cluster/data/hg17/bed/geneSorter/blastp/mm7
mkdir run
cd run
mkdir out

# Make blast script 
# NOTE!! left off " b 1" from the end of the script because
#  we wanted to be able to get the near-best, not just the best one.
cat  << '_EOF_' > blastSome
#!/bin/sh
BLASTMAT=/cluster/bluearc/blast229/data /cluster/bluearc/blast229/blastall \
-p blastp -d /cluster/panasas/home/store/mm7/blastp/known \
-i $1 -o $2 -e 0.001 -m 8
'_EOF_'
    # << keep emacs happy
chmod a+x blastSome

# Make gensub2 file
cat << '_EOF_' > gsub
#LOOP
blastSome {check in line+ $(path1)} {check out line out/$(root1).tab}
#ENDLOOP
'_EOF_'
    # << keep emacs happy

# Create parasol batch
#       this echo trick is used because otherwise the command line is
#       too long and you can not do a simple ls
echo ../../split/*.fa | wordLine stdin > split.lst
gensub2 split.lst single gsub jobList
para create jobList
para try
para check
para push ... etc ...

Completed: 7735 of 7735 jobs
CPU time in finished jobs:      97096s    1618.26m    26.97h    1.12d  0.003 y
IO & Wait Time:                564656s    9410.94m   156.85h    6.54d  0.018 y
Average job time:                  86s       1.43m     0.02h    0.00d
Longest running job:                0s       0.00m     0.00h    0.00d
Longest finished job:             240s       4.00m     0.07h    0.00d
Submission to last job:          1272s      21.20m     0.35h    0.01d


# Load into database.
ssh hgwdev
cd /cluster/data/hg17/bed/geneSorter/blastp/mm7/run/out
hgLoadBlastTab hg17 mmBlastTab -maxPer=1 *.tab
Scanning through 7735 files
Loading database with 33306 rows

# changed mm6 to mm7 in src/hg/hgGene/hgGeneData/Human/hg17/otherOrgs.ra
# and checked it in.

# hgLoadBlastTab hg17 mmBlastTabTopN -maxPer=250 *.tab
#  (not done, this was only used for research)

# hgLoadBlastTab hg17 mmBlastNearBest -topPercent=5 *.tab > hgMmNearBest.stats
#  (this will be the new way to go)
Reading seq lengths from hg17.knownGenePep
Finding max gene combined-coverage scores in 7735 files
Scanning through 7735 files
Loading database with 51520 rows


##########################################################################
# MYTOUCH FIX - jen - 2006-01-24
  sudo mytouch hg17 gencodeGeneClassJun05 0508301200.00
  note - gencodeGeneClassJun05 table on dev only
  sudo mytouch hg17 knownGeneLink 0506050000.00
  sudo mytouch hg17 ensGtp 0505241200.00
  sudo mytouch hg17 ccdsInfo 0505241200.00


##########################################################################
# BLASTZ OPOSSUM monDom2 (WORKING - 2006-01-23 - Hiram)

    ssh kk
    #	running out of disk space on store5:
    [hiram@kk /cluster/data/hg17/bed] df -h .
    #Filesystem            Size  Used Avail Use% Mounted on
    #			   1.5T  1.3T   79G  95% /cluster/store5
    #	So, keep this elsewhere, and symlink it:
    cd /cluster/data/hg17/bed
    ln -s /cluster/store9/hg17/bed/blastzMonDom2.2006-01-23 \
	./blastzMonDom2.2006-01-23
    ln -s blastzMonDom2.2006-01-23 blastz.monDom2

    cd /cluster/data/hg17/bed/blastzMonDom2.2006-01-23

    cat << '_EOF_' > DEF
# human vs. opossum
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/i386:/parasol/bin

BLASTZ=blastz.v7

# Specific settings for chicken (per Webb email to Brian Raney)
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=10000
BLASTZ_K=2200
BLASTZ_Q=/cluster/data/blastz/HoxD55.q
BLASTZ_ABRIDGE_REPEATS=0

# TARGET: Human (hg17)
SEQ1_DIR=/scratch/hg/hg17/bothMaskedNibs
SEQ1_LEN=/cluster/data/hg17/chrom.sizes
SEQ1_IN_CONTIGS=0
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Opossum monDom2
SEQ2_DIR=/scratch/hg/monDom2/monDom2.2bit
SEQ2_LEN=/scratch/hg/monDom2/chrom.sizes
SEQ2_IN_CONTIGS=0
SEQ2_CHUNK=30000000
SEQ2_LIMIT=100
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/blastzMonDom2.2006-01-23
TMPDIR=/scratch/tmp
'_EOF_'
    #	<< happy emacs

    cd /cluster/data/hg17/bed/blastzMonDom2.2006-01-23
    time /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=kk -chainMinScore=5000 -chainLinearGap=loose \
	`pwd`/DEF > blastz.out 2>&1 &

    #	real    1122m44.191s
    #	failed during the load of chr19
    #	hgLoadChain hg17 chr19_chainMonDom2 chr19.chain
    #	Out of memory needMem - request size 56 bytes

    #	So, go to kolossus:
    ssh kolossus
    #	There isn't any hg17 db here yet, get it established with a
    #	chromInfo and a 2bit sequence:
    hgsql -e "create database hg17;" mysql
    cd /cluster/data/hg17
    twoBitInfo hg17.2bit stdout |
        awk '{printf "%s\t%s\t/gbdb/hg17/hg17.2bit\n", $1,$2}' \
		> chromInfo.kolossus.tab
    hgsql hg17 < $HOME/kent/src/hg/lib/chromInfo.sql
    hgsql hg17 \
-e 'load data local infile "chromInfo.kolossus.tab" into table chromInfo;'
    #	it appears /gbdb/hg17 already exists
    ln -s /cluster/data/hg17/hg17.2bit /gbdb/hg17/hg17.2bit
    #	now, loading only chr19:
    cd /cluster/data/hg17/bed/blastzMonDom2.2006-01-23/axtChain
    hgLoadChain hg17 chr19_chainMonDom2 chain/chr19.chain
    #	real    33m31.689s
    #	while that is running, back on hgwdev, get the other chains loaded
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastzMonDom2.2006-01-23/axtChain
    cp loadUp.csh loadUp.noChr19.csh
    #	change the foreach line to eliminate the chr19.chain:
    diff loadUp.csh loadUp.noChr19.csh
    < foreach f (*.chain)
    ---
    > foreach f (`ls *.chain | grep -v chr19.chain`)
    #	And then run that script
    time ./loadUp.noChr19.csh > load.noChr19.out 2>&1
    #	real    76m8.757s
    
    #	When the kolossus load finishes, email to push-request and ask
    #	for the two tables to be pushed from kolossus to hgwdev:
    #	chr19_chainMonDom2
    #	chr19_chainMonDom2Link

    #	then, continuing:
    time /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-continue=download -bigClusterHub=pk -chainMinScore=5000 \
	-chainLinearGap=loose `pwd`/DEF > download.out 2>&1 &
    #	real    2m42.505s

    #	now, back on kolossus to run a featurebits
    time featureBits hg17 chainMonDom2Link >fb.hg17.chainMonDom2Link 2>&1
    #	355119482 bases of 2866216770 (12.390%) in intersection
    featureBits hg17 chainMonDom1Link
    #	456069062 bases of 2866216770 (15.912%) in intersection

    #	Then, to swap the results:
    ssh kk
    cd /cluster/data/hg17/bed/blastz.monDom2
    time /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-swap -bigClusterHub=pk -chainMinScore=5000 \
	-chainLinearGap=loose `pwd`/DEF > swap.out 2>&1 &
    #	running 2006-01-30 11:25
    #	real    47m27.082s
    #	failed during the load - as with the Hg18 experiment, something
    #	is really huge about these results.

#########################################################################
# BUILD MAF ANNOTATION FOR MULTIZ17WAY  (kate 2006-02-16)
# Redo to fix overlaps (2006-04-09 kate)
# rebuild frames to get bug fix, using 1-pass maf methodology (2006-06-09 markd)

    ssh kkstore01
    cd /cluster/data/rheMac2
    twoBitInfo -nBed rheMac2.2bit rheMac2.N.bed

    ssh kkstore02
    cd /cluster/data/hg17/bed/multiz17way
    mkdir anno 
    cd anno
    mkdir maf run
    cd run
    rm sizes nBeds
    foreach i (`cat /cluster/data/hg17/bed/multiz17way/species.lst`)
        ln -s  /cluster/data/$i/chrom.sizes $i.len
        ln -s  /cluster/data/$i/$i.N.bed $i.bed
        echo $i.bed  >> nBeds
        echo $i.len  >> sizes
    end

    rm jobs.csh
    echo date > jobs.csh
    foreach i (../../maf/*.maf)
        echo nice mafAddIRows -nBeds=nBeds -sizes=sizes $i /cluster/data/hg17/hg17.2bit ../maf/`basename $i` >> jobs.csh
        echo "echo $i" >> jobs.csh
    end 
    echo date >> jobs.csh

    # do smaller jobs first
    tac jobs.csh > jobsRev.csh
    mv jobsRev.csh jobs.csh
    
    csh jobs.csh >&! jobs.log &
        # 1.5 hrs.
        # 9 hours for redo -- something wrong ?

    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz17way/anno/maf
    mkdir -p /gbdb/hg17/multiz17way/anno/maf
    ln -s /cluster/data/hg17/bed/multiz17way/anno/maf/*.maf \
        /gbdb/hg17/multiz17way/anno/maf
cat > loadMaf.csh << 'EOF'
    date
    hgLoadMaf -pathPrefix=/gbdb/hg17/multiz17way/anno/maf \
                hg17 multiz17way
    date
'EOF'
    csh loadMaf.csh >&! loadMaf.log &

    # load summary table on kolossus, as it crashes on hgwdev
    ssh kolossus
    cd /cluster/data/hg17/bed/multiz17way/anno/maf
    cat *.maf | \
        nice hgLoadMafSummary hg17 -minSize=30000 -mergeGap=1500 \
            -maxSize=200000  multiz17waySummary stdin

        # Created 3212623 summary blocks from 114139253 components and 17522217 mafs from stdin
    # request push to hgwdev

    # Dropped unused indexes (2006-05-09 kate)
    # NOTE: this is not required in the future, as the loader
    # has been fixed to not generate these indexes
    hgsql hg17 -e "alter table multiz17waySummary drop index chrom_2"
    hgsql hg17 -e "alter table multiz17waySummary drop index chrom_3"

    ssh kkstore02
    cd /cluster/data/hg17/bed/multiz17way
    set sanDir = /san/sanvol1/scratch/hg17/multiz17way/frames
    mkdir -p $sanDir/maf
    cp -rp maf/* $sanDir/maf
    mkdir frames
    cd frames
    cp /cluster/data/mm7/bed/multiz17wayFrames/mkMafFrames .
    cp /cluster/data/mm7/bed/multiz17wayFrames/Makefile .
    #edit Makefile to correct species names and set and sanDir

    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz17way/frames
    make getGenes >&! getGenes.log &
        # ~1 minute
    make getFrames >&! getFrames.log &
        # ~2 hours
    # NOTE: if jobs get hung up (e.g. running for hours, when
    #   they should run for minutes, do 'para stop' so that
    #   the 'para make' can restart the job
    make loadDb >&! loadDb.log &

    ###
    # rebuild frames to get bug fix, using 1-pass maf methodology
    # (2006-06-09 markd)
    ssh kkstore02
    cd /cluster/data/hg17/bed/multiz17way/frames
    mv mafFrames/ mafFrames.old
    nice tcsh # easy way to get process niced
    (cat  ../maf/*.maf | genePredToMafFrames hg17 stdin stdout bosTau2 genes/bosTau2.gp.gz canFam2 genes/canFam2.gp.gz danRer3 genes/danRer3.gp.gz fr1 genes/fr1.gp.gz galGal2 genes/galGal2.gp.gz hg17 genes/hg17.gp.gz mm7 genes/mm7.gp.gz oryCun1 genes/oryCun1.gp.gz panTro1 genes/panTro1.gp.gz rheMac2 genes/rheMac2.gp.gz rn3 genes/rn3.gp.gz xenTro1 genes/xenTro1.gp.gz |  gzip >multiz17way.mafFrames.gz)>&log&
    ssh hgwdev
    cd /cluster/data/hg17/bed/multiz17way/frames

    hgLoadMafFrames hg17 multiz17wayFrames multiz17way.mafFrames.gz >&log&


# EXTRACT LINEAGE-SPECIFIC REPEATS FOR RAT (DONE 2/8/06 angie)
    ssh kolossus
    mkdir /cluster/data/hg17/rmsk
    cd /cluster/data/hg17/rmsk
    ln -s ../*/chr*.fa.out .
    # Run Arian's DateRepsinRMoutput.pl to add extra columns telling 
    # whether repeats in -query are also expected in -comp species.  
    # Even though we already have the human-mouse linSpecReps,
    # extractLinSpecReps requires two columns of DateRepsinRMoutput.pl
    # additions.  So add mouse, then ignore it.  
    # Rat in extra column 1, Mouse in extra column 2
    foreach outfl ( *.out )
        echo "$outfl"
        /cluster/bluearc/RepeatMasker/DateRepeats \
          ${outfl} -query human -comp rat -comp mouse
    end
    # Now extract rat (extra column 1), ignore mouse.
    cd ..
    mkdir linSpecRep.notInRat
    foreach f (rmsk/*.out_rat*_mus-musculus)
        set base = $f:t:r:r
        echo $base.out.spec
        /cluster/bin/scripts/extractLinSpecReps 1 $f > \
                        linSpecRep.notInRat/$base.out.spec
    end
    # Distribute and clean up.
    rsync -av linSpecRep.notInRat /san/sanvol1/scratch/hg17/
    rm -r rmsk


# BLASTZ/CHAIN/NET RN4 (DONE 2/10/06 angie)
    ssh kkstore01
    mkdir /cluster/data/hg17/bed/blastz.rn4.2006-02-08
    cd /cluster/data/hg17/bed/blastz.rn4.2006-02-08
    cat << '_EOF_' > DEF
# human vs. rat

BLASTZ_ABRIDGE_REPEATS=1

# TARGET: Human
SEQ1_DIR=/san/sanvol1/scratch/hg17/nib
SEQ1_SMSK=/san/sanvol1/scratch/hg17/linSpecRep.notInRat
SEQ1_LEN=/cluster/data/hg17/chrom.sizes
SEQ1_CHUNK=10000000
SEQ1_LAP=10000

# QUERY: Rat
SEQ2_DIR=/san/sanvol1/scratch/rn4/nib
SEQ2_SMSK=/san/sanvol1/scratch/rn4/linSpecRep.notInHuman
SEQ2_LEN=/cluster/data/rn4/chrom.sizes
SEQ2_CHUNK=10000000
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/blastz.rn4.2006-02-08
'_EOF_'
    # << for emacs
    doBlastzChainNet.pl DEF -chainLinearGap medium \
      -bigClusterHub pk -smallClusterHub pk -workhorse pk \
      -blastzOutRoot /san/sanvol1/scratch/blastzHg17Rn4Out >& do.log &
    tail -f do.log
    rm -f /cluster/data/hg17/bed/blastz.rn4
    ln -s blastz.rn4.2006-02-08 /cluster/data/hg17/bed/blastz.rn4

# UPDATE WGRNA TRACK (DONE, 2006-02-15, Fan)

  ssh hgwdev
  cd /cluster/data/hg17/bed

  mkdir wgRna-2006-02-15
  cd wgRna-2006-02-15

# Received the data file, wg_track_hg17_feb2006_completed.txt, from Michel Weber's email
# (Michel.Weber@ibcg.biotoul.fr)
# and place it under cd /cluster/data/hg17/bed/wgRna-2006-02-15.

  cp -p wg_track_hg17_feb2006_completed.txt wgRna.tab
  hgLoadBed -sqlTable=/cluster/home/fanhsu/src/hg/lib/wgRna.sql hg17 wgRna wgRna.tab

# Compared to previous data, 2 records deleted, 27 records added.


########################################################################
#  BLASTZ Opossum monDom4 (DONE - 2006-02-21 - 2006-02-26 - Hiram)
    ssh pk
    mkdir /cluster/data/hg17/bed/blastzMonDom4.2006-02-21
    cd /cluster/data/hg17/bed
    ln -s blastzMonDom4.2006-02-21 blastz.monDom4
    cd blastzMonDom4.2006-02-21

    cat << '_EOF_' > DEF
# human vs. opossum
export PATH=/usr/bin:/bin:/usr/local/bin:/cluster/bin/penn:/cluster/bin/x86_64:/parasol/bin

BLASTZ=blastz.v7.x86_64

# settings for more distant organism alignments
BLASTZ_H=2000
BLASTZ_Y=3400
BLASTZ_L=10000
BLASTZ_K=2200
BLASTZ_M=50
BLASTZ_Q=/cluster/data/blastz/HoxD55.q

# TARGET: Human (hg17)
SEQ1_DIR=/scratch/hg/hg17/bothMaskedNibs
SEQ1_LEN=/cluster/data/hg17/chrom.sizes
SEQ1_CHUNK=50000000
SEQ1_LAP=10000

# QUERY: Opossum monDom4
SEQ2_DIR=/san/sanvol1/scratch/monDom4/monDom4.2bit
SEQ2_LEN=/san/sanvol1/scratch/monDom4/chrom.sizes
SEQ2_CHUNK=30000000
SEQ2_LAP=0

BASE=/cluster/data/hg17/bed/blastzMonDom4.2006-02-21
TMPDIR=/scratch/tmp
'_EOF_'
    #  << happy emacs

    time /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk -chainMinScore=5000 -chainLinearGap=loose \
	-continue=chainMerge `pwd`/DEF > chainMerge.out 2>&1 &
    time /cluster/bin/scripts/doBlastzChainNet.pl -verbose=2 \
	-bigClusterHub=pk -chainMinScore=5000 -chainLinearGap=loose \
	-swap `pwd`/DEF > swap.out 2>&1 &

    time nice -n +19 featureBits hg17 chainMonDom4Link \
	> fb.hg17.chainMonDom4Link 2>&1
    time nice -n +19 featureBits monDom4 chainHg17Link \
	> fb.monDom4.chainHg17Link 2>&1

########################################################################
##  Measuring MonDom4 chain pile ups (DONE - 2006-02-26 - Hiram)
    ssh kkstore02
    cd /cluster/data/hg17/bed/blastz.monDom4/axtChain
    #	extract coordinates on the target genome of the chains
    zcat hg17.monDom4.all.chain.gz | grep "^chain " \
    | awk '{printf "%s\t%s\t%s\t%s\t%s\n", $3, $6, $7, $5, $2}' \
        | gzip -c > target.chain.bed.gz
    # turn that into a wiggle graph with bedItemOverlapCount
    # use HGDB_CONF for read-only access to the hg17 DB in bedItemOverlapCount
    #	it wants to read chromInfo ...
    export HGDB_CONF=~/.hg.conf.read-only
    #	ignore chains longer than 1,000,000
    zcat target.chain.bed.gz | awk '$3-$2<1000000 {print}' \
	| sort -k1,1 -k2,2n \
	    | bedItemOverlapCount hg17 stdin \
		| wigEncode stdin monDom4PileUps.wig monDom4PileUps.wib
    #	Do the same for the query coordinates to find out where these
    #	chains are coming from
    zcat hg17.monDom4.all.chain.gz | grep "^chain " \
    | awk '{printf "%s\t%s\t%s\t%s\t%s\n", $8, $11, $12, $10, $2}' \
        | gzip -c > query.chain.bed.gz
    zcat query.chain.bed.gz | awk '$3-$2<1000000 {print}' \
    | sort -k1,1 -k2,2n \
        | bedItemOverlapCount monDom4 stdin \
            | wigEncode stdin hg17PileUps.wig hg17PileUps.wib
    #	load those wiggles
    ssh hgwdev
    cd /cluster/data/hg17/bed/blastz.monDom4/axtChain
    ln -s `pwd`/monDom4PileUps.wib /gbdb/hg17/wib
    ln -s `pwd`/hg17PileUps.wib /gbdb/monDom4/wib

    hgLoadWiggle -verbose=2 hg17 monDom4PileUp monDom4PileUps.wig
    hgLoadWiggle -verbose=2 monDom4 hg17PileUps hg17PileUps.wig
    #	add wiggle track type entries to the respective trackDb.ra files



# UPDATE hg17 knownToVisiGene (2006-03-07 galt)
# Create table that maps between known genes and visiGene database 
    # mapping to other species such as mouse, zebrafish, frog
    # requires visiGene probe track vgImageProbes be created first
    knownToVisiGene hg17 -fromProbePsl=vgImageProbes

############################################################################
# Add Landmark track (2006-03-08 giardine)
# Note: This track is for regulatory regions and other landmarks that are not
    #included in other tracks.  It is being gathered from the locus experts
    #that are contributing data to the Human Mutation track.  This should
    #be helpful in understanding the data in the mutation track.
#table definitions for autoSql
autoSql landmark.as landmark -dbLink
#change index on bin to normal index not primary key
#move bin in struct so works as bed 4+
#copy autoSql files to hg/lib and hg/inc (add .o file to makefile)
#cat together landmark files from sources in landmark.bed then sort
grep "^chr" landmark.bed | sort -k1,1 -k2,2n > sortedLandmark.bed
#loading
hgsql hg17 < landmark.sql
hgLoadBed hg17 landmark sortedLandmark.bed -noSort -oldTable -tab
#add to trackDb.ra file (human hg17 level)

#changed landmark track to provide links and attributes in prep for ORegAnno
#data.  Got set of test data by grabbing their .gff file used for custom
#tracks and converting to bed, then to landmarks format.
cd humPhen/landmarkData/June06/
#convert data to new formats then
cat newLandmark.txt landmarkORA.txt  > allLandmarks.txt
grep "^chr" allLandmarks.txt | sort -k1,1 -k2,2n > sortedAllLandmark.txt
#start new tables
cd humPhen/kent/src/hg/lib/
autoSql landmark.as landmark -dbLink
#move bin in .h file to end of structure, to make load work
mv landmark.h ../inc/landmark.h
#change primary key to indexes where not unique, add index on landmarkId
#limit name, landmarkType, raKey size to 64
hgsql -e "drop table landmark;" hg17
hgsql hg17 < landmark.sql
cd ~giardine/humPhen/landmarkData/June06/
hgLoadBed hg17 landmark sortedAllLandmark.txt -noSort -oldTable -tab
hgsql hg17
        load data local infile "landmarkAttrORA.txt" into table landmarkAttr;
        load data local infile "landmarkAttrLinkORA.txt" into table landmarkAttrLink;
        load data local infile "landmarkAttrCat.txt" into table landmarkAttrCat;
cd ../../kent/src/
make clean
make libs
cd hg
make cgi
cd makeDb/trackDb
make DBS=hg17 update
#test in hgwdev-giardine

#redo landmarks, moving categories out of database
convertORAformat < ORegAnnoBed
#start new tables
cd humPhen/kent/src/hg/lib/
autoSql landmark.as landmark -dbLink
#move bin in .h file to end of structure, to make load work
mv landmark.h ../inc/landmark.h
#change primary key to indexes, add primary key on landmarkId
#limit name, landmarkType, raKey size to 64
#only need to reload attributes rest of data & tables same
hgsql -e "drop table landmarkAttr;" hg17
hgsql -e "drop table landmarkAttrCat;" hg17
cd ../../../../landmarkData/June06/
hgsql hg17
        #cut and paste in create table landmarkAttr
        load data local infile "landmarkAttrORA.txt" into table landmarkAttr;
#Records: 2028  Deleted: 0  Skipped: 0  Warnings: 8 ???
cd ../../kent/src/
make clean
make libs
cd hg
make cgi
cd makeDb/trackDb
make DBS=hg17 update
#test in hgwdev-giardine


############################################################################
# hg15 -> hg17 LIFTOVER CHAINS (STARTED 3/9/06, DONE 3/10/06 Fan)
    # I used a size of 10kb instead of 3kb for the split (blat query) sizes in
    # hg17.  This had a huge affect on the amount of hits in the blat, which
    # then had a huge effect on the amount of chains.  I should also mention 
    # that hg17 chromosomes chr1 and chr2 were split further 
    # into more than a single query file.  This helped a LOT in avoiding 
    # cluster hippos classically associated with those chroms.
    
    ######## LIFTOVER PREPARATION
    # Split up hg17
    ssh pk
    cd /san/sanVol1/scratch/hg17
    mkdir -p liftSplits/{split,lift}
    bash
    for fa in /cluster/data/hg17/?{,?,*hap*}/*.fa; do
      c=`basename $fa .fa`
      echo $c
      faSplit -lift=liftSplits/lift/${c}.lft size $fa -oneFile 10000 liftSplits/split/$c
    done
    mkdir -p biggerSplits/split
    cd biggerSplits/
    ln -s ../liftSplits/lift
    cd split/
    ln -s ../../liftSplits/split/* .
    faSplit sequence chr1.fa 5 chr1_
    faSplit sequence chr2.fa 5 chr2_
    rm chr{1,2}.fa

    # Make some dirs
    cd /san/sanVol1/scratch
    mkdir -p hg15

    # Copy 11.ooc files to hg15 subdirectory.
    cp -p /cluster/store5/gs.16/build33/11.ooc hg15

    ## First, copy over scripts. (Already done before)

    # mkdir -p /san/sanVol1/scratch/fan
    # cp -p /san/sanVol1/scratch/fan/*.sh /san/sanVol1/scratch/fan
    # cp /san/sanVol1/scratch/andy/psl.header /san/sanVol1/scratch/fan

    ######## LIFTOVER BLATTING  

    # HG15
    ssh pk
    cd /cluster/data/hg15
#    makeLoChain-align hg15 /scratch/hg/hg15/bothMaskedNibs hg17 \     

    makeLoChain-align hg15 /scratch/hg/hg15/chromTrfMixedNib hg17 \
/san/sanVol1/scratch/hg17/biggerSplits/split
    cd bed

    mv blat.hg17.2006-03-09 /san/sanVol1/scratch/hg15 
    cd /san/sanVol1/scratch/hg15/blat.hg17.2006-03-09/run/
    sed 's/^blat/blat.sh/; s/\}.*$/}/' spec | awk '{print "/san/sanVol1/scratch/fan/" $0 " hg15ToHg17"}' > newspec
    para create newspec
    para try
    para push
# Saw some failures, keep pushing again, they finally all finished.  
# The problems were all from one node.  
# Used "para remove machine ..." to remove that node from the cluster.
# Completed: 2376 of 2376 jobs
# CPU time in finished jobs:     626355s   10439.25m   173.99h    7.25d  0.020 y
# IO & Wait Time:                 49512s     825.20m    13.75h    0.57d  0.002 y
# Average job time:                 284s       4.74m     0.08h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:            3693s      61.55m     1.03h    0.04d
# Submission to last job:          4165s      69.42m     1.16h    0.05d

    ######## LIFTOVER CHAINING
    # LIFTING
    ssh pk
    cd /san/sanVol1/scratch/fan
    cp mm7SplitLift.sh hg17SplitLift.sh

    # change andy to fan, mm7 to hg17, and chrX to chr2, and remove chrUn_random 
    vi hg17SplitLift.sh

    cat << 'EOF' > hg17ChainMergeSplit.sh
#!/bin/bash
cp -r chainRaw/ /scratch/fan/hg17Lifts
pushd /scratch/fan/hg17Lifts
mkdir chain
/cluster/bin/x86_64/chainMergeSort chainRaw/*.chain | /cluster/bin/x86_64/chainSplit chain stdin
cp -r chain `dirs +1`
rm -rf chain chainRaw
'EOF'

    chmod +x hg17ChainMergeSplit.sh

    # HG15
    cd /san/sanVol1/scratch/hg15/blat.hg17.2006-03-09/raw
    /san/sanVol1/scratch/fan/hg17SplitLift.sh
    cd ../    
    mkdir chainRun chainRaw
    cd chainRun
    cat > gsub << 'EOF'
#LOOP
/cluster/bin/x86_64/axtChain -verbose=0 -linearGap=medium -psl $(path1) /scratch/hg/hg15/chromTrfMixedNib  /san/sanVol1/scratch/hg17/nib {check out line+ ../chainRaw/$(root1).chain}
#ENDLOOP
'EOF'
    ls -1S ../psl/*.psl > in.lst
    gensub2 in.lst single gsub spec
    para create spec
    para push
    para time
# Completed: 46 of 46 jobs
# CPU time in finished jobs:       3546s      59.10m     0.98h    0.04d  0.000 y
# IO & Wait Time:                   895s      14.92m     0.25h    0.01d  0.000 y
# Average job time:                  97s       1.61m     0.03h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:             270s       4.50m     0.07h    0.00d
# Submission to last job:           270s       4.50m     0.07h    0.00d

    ######### CHAINMERGE/NET/NETSUBSET
    ssh kolossus
    mkdir -p /scratch/fan/hg17Lifts
    cd /scratch/fan/hg17Lifts

    cp -r /san/sanvol1/scratch/hg15/blat.hg17.2006-03-09/chainRaw/ .
    mkdir chain
    /cluster/bin/x86_64/chainMergeSort chainRaw/* | /cluster/bin/x86_64/chainSplit chain stdin
# about 30 minutes.

    cp -rp chain /san/sanvol1/scratch/hg15/blat.hg17.2006-03-09/
    rm -rf chain
    rm -rf chainRaw

    ssh pk
    cd /san/sanvol1/scratch/fan
    cat << 'EOF' > netOver.sh 
#!/bin/bash

chain=$1
chrom=`basename $chain .chain`
sizesHGOld=$2
sizesHG17=/cluster/data/hg17/chrom.sizes
chainDir=`dirname $chain`
blatDir=`dirname $chainDir`
net=${blatDir}/net/${chrom}.net
over=${blatDir}/over/${chrom}.over

mkdir -p ${blatDir}/{over,net}
/cluster/bin/x86_64/chainNet $chain $sizesHGOld $sizesHG17 $net /dev/null
/cluster/bin/x86_64/netChainSubset $net $chain $over
'EOF'
# << emacs
    chmod +x netOver.sh

    mkdir netRun

    cd netRun/

    find /san/sanVol1/scratch/hg15/blat.hg17.2006-03-09/chain -name "*.chain" \
     | awk '{print "/san/sanVol1/scratch/fan/netOver.sh " $1 " /cluster/data/hg15/chrom.sizes"}' >> spec
    para create spec
    para push
    para time
# Completed: 44 of 44 jobs
# CPU time in finished jobs:        427s       7.12m     0.12h    0.00d  0.000 y
# IO & Wait Time:                   248s       4.13m     0.07h    0.00d  0.000 y
# Average job time:                  15s       0.26m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              29s       0.48m     0.01h    0.00d
# Submission to last job:            46s       0.77m     0.01h    0.00d

# seems much faster than mm7.

    ########## FINISHING
    ssh hgwdev

    # HG15
    cd /san/sanvol1/scratch/hg15/blat.hg17.2006-03-09/over
    cat * >> ../hg15ToHg17.over.chain
    cd ../
    rm -rf psl/ net/ chain/ chainRaw/ over/
    cd ../
    cp -rp blat.hg17.2006-03-09/ /cluster/data/hg15/bed

    cd /cluster/data/hg15/bed
    ln -s blat.hg17.2006-03-09 blat.hg17
    ln -s `pwd`/blat.hg17/hg15ToHg17.over.chain liftOver/hg15ToHg17.over.chain
    ln -s `pwd`/liftOver/hg15ToHg17.over.chain /gbdb/hg15/liftOver/hg15ToHg17.over.chain
    mkdir -p /usr/local/apache/htdocs/goldenPath/hg15/liftOver
    cd /usr/local/apache/htdocs/goldenPath/hg15/liftOver
    cp /gbdb/hg15/liftOver/hg15ToHg17.over.chain .
    gzip hg15ToHg17.over.chain
    hgAddLiftOverChain hg15 hg17 /gbdb/hg15/liftOver/hg15ToHg17.over.chain

# UPDATED hg17.knownToVisiGene (2006-03-14 galt)
#  after making sure hg17.vgAllProbes was up to date (see makeVisiGene.doc)
ssh hgwdev
knownToVisiGene hg17 -fromProbePsl=vgAllProbes

########################################################################
### microRNA targets tracks  (DONE - 2006-03-17 - 2006-04-27 - Hiram)
### from: http://pictar.bio.nyu.edu/ Rajewsky Lab
### Nikolaus Rajewsky nr@scarbo.bio.nyu.edu
### Yi-Lu Wang ylw205@nyu.edu
### dg@thp.Uni-Koeln.DE
    ssh hgwdev
    mkdir /cluster/data/hg17/bed/picTar
    cd /cluster/data/hg17/bed/picTar
    wget --timestamping \
	'http://pictar.bio.nyu.edu/ucsc/new_mammals_bed' -O newMammals.bed

    wget --timestamping \
	'http://pictar.bio.nyu.edu/ucsc/new_mammals_chicken_bed' \
	-O newMammalsChicken.bed

    grep -v "^track" newMammals.bed \
	| hgLoadBed -strict hg17 picTarMiRNA4Way stdin
    #	Loaded 205263 elements of size 9
    grep -v "^track" newMammalsChicken.bed \
	| hgLoadBed -strict hg17 picTarMiRNA5Way stdin
    #	Loaded 43081 elements of size 9

    nice -n +19 featureBits hg17 picTarMiRNA4Way
    #	608549 bases of 2866216770 (0.021%) in intersection
    nice -n +19 featureBits hg17 picTarMiRNA5Way
    #	109059 bases of 2866216770 (0.004%) in intersection

############################################################################
# dbSNP BUILD 125 (Heather, March 2006)

# Set up directory structure
ssh kkstore02
cd /cluster/data/dbSnp
mkdir 125
cd 125
mkdir shared
mkdir shared/data
mkdir shared/schema
mkdir organisms
mkdir organisms/human_9606
mkdir organisms/human_9606/rs_fasta
mkdir organisms/human_9606/database
mkdir organisms/human_9606/database/organism_data
mkdir organisms/human_9606/database/organism_data/hg17
mkdir organisms/human_9606/database/schema

# Get data from NCBI

# Shared data includes data dictionary, 
# Shared data includes defined types such as validity, class, function, locType
# Actually this is independent of hg17 build and should go in separate makeDoc
cd shared/data
ftp ftp.ncbi.nih.gov
cd snp/database/organism_shared_data
mget *.gz

cd ../schema
ftp ftp.ncbi.nih.gov
cd snp/database/schema/shared_schema
mget *.gz

# using headers of fasta files for molType, class and observed
cd ../organisms/human_9606/rs_fasta
ftp ftp.ncbi.nih.gov
cd snp/organisms/human_9606/rs_fasta
mget *.gz

cd ../database/organism_data/hg17
ftp ftp.ncbi.nih.gov
cd snp/organisms/human_9606/database/organism_data
# ContigLoc table has coords, orientation, loc_type, and refNCBI allele
get b125_SNPContigLoc_35_1.bcp.gz
# ContigLocusId has function
get b125_SNPContigLocusId_35_1.bcp.gz
get b125_ContigInfo_35_1.bcp.gz
# MapInfo has alignment weights
get b125_SNPMapInfo_35_1.bcp.gz
# SNP has validation status and heterozygosity
get SNP.bcp.gz
# done with FTP

# rename
mv b125_SNPContigLoc_35_1.bcp.gz ContigLoc.gz
mv b125_SNPContigLocusId_35_1.bcp.gz ContigLocusId.gz
mv b125_ContigInfo_35_1.bcp.gz ContigInfo.gz
mv b125_SNPMapInfo_35_1.bcp.gz MapInfo.gz
mv SNP.bcp.gz SNP.gz

# edit table descriptions
cd /cluster/data/dbSnp/125/organisms/human_9606/database/schema
# get CREATE statements from human_9606_table.sql for our 5 tables
# store in table.tmp
# convert and rename tables
sed -f 'mssqlToMysql.sed' table.tmp > table2.tmp
rm table.tmp
sed -f 'tableRename.sed' table2.tmp > table.sql
rm table2.tmp

# get header lines from rs_fasta
cd /cluster/data/dbSnp/125/organisms/human_9606/rs_fasta
/bin/csh gnl.csh

# load on kkr5u00
ssh kkr5u00
hgsql -e mysql 'create database dbSnpHumanBuild125' 
cd /cluster/data/dbSnp/125/organisms/human_9606/database/schema
hgsql dbSnpHumanBuild125 < table.sql
cd ../organism_data/hg17
/bin/csh load.csh

# note rowcount
# ContigLoc     24135144 
# SNP           10430754 
# MapInfo       10271016 
# ContigLocusId  9539145 

# create working /scratch dir
cd /scratch/snp
mkdir 125
cd 125
mkdir human
cd human

# get hg17 ctgPos, load into dbSnpHumanBuild125, compare contig list between ctgPos and ContigInfo

# get gnl files
cp /cluster/data/dbSnp/125/organisms/human_9606/rs_fasta/*.gnl .

# examine ContigInfo for group_term and edit pipeline.csh
# use "ref_haplotype" 

# filter ContigLoc into ContigLocFilter
# this gets rid of alternate assemblies and poor quality alignments
# uses ContigInfo and MapInfo (weight == 10 || weight == 3)
# assumes all contigs are positively oriented
# will abort if not true

mysql> desc ContigLocFilter;
#  +---------------+-------------+------+-----+---------+-------+
#  | Field         | Type        | Null | Key | Default | Extra |
#  +---------------+-------------+------+-----+---------+-------+
#  | snp_id        | int(11)     | NO   |     |         |       |
#  | ctg_id        | int(11)     | NO   |     |         |       |
#  | chromName     | varchar(32) | NO   |     |         |       |
#  | loc_type      | tinyint(4)  | NO   |     |         |       |
#  | phys_pos_from | int(11)     | NO   |     |         |       |
#  | phys_pos      | varchar(32) | YES  |     | NULL    |       |
#  | orientation   | tinyint(4)  | NO   |     |         |       |
#  | allele        | blob        | YES  |     | NULL    |       |
#  +---------------+-------------+------+-----+---------+-------+
 
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpContigLocFilter dbSnpHumanBuild125 ref_haplotype
# note rowcount
# ContigLocFilter  10113426
# how many are positive strand? hopefully 90%
mysql> select count(*) from ContigLocFilter where orientation = 0;
# 9161012

# filter ContigLocusId into ContigLocusIdFilter
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpContigLocusIdFilter dbSnpHumanBuild125 ref_haplotype
# note rowcount 
# ContigLocusIdFilter  5352542

# condense ContigLocusIdFilter into ContigLocusIdCondense (one SNP can have multiple functions)
# assumes SNPs are in numerical order
# will errAbort if not true
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpContigLocusIdCondense dbSnpHumanBuild125 
# note rowcount
# expect about 50% for human
# ContigLocusIdCondense 4129899
# could delete ContigLocusIdFilter table here

# create chrN_snpFasta tables from *.gnl files
# snpLoadFasta.error will report all SNPs with "lengthTooLong"
# here we have 4428 SNPs with lengthTooLong
# these are noted as ObservedNotAvailable
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpLoadFasta dbSnpHumanBuild125 

# split ContigLocFilter by chrom (could start using pipeline.csh here)
# pipeline.csh takes about 35 minutes to run
# create the first chrN_snpTmp
# we will reuse this table name, adding/changing columns as we go
# at this point chrN_snpTmp will have the same description as ContigLocFilter
# this opens a file handle for every chrom, so will not scale to scaffold-based assemblies
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpSplitByChrom dbSnpHumanBuild125 ref_haplotype

# generate true coords using loc_type
# possible errors logged to snpLocType.error:
# "Missing quotes in phys_pos for range"
# "Chrom end <= chrom start for range"
# "Wrong size for exact"
# "Unknown locType"
# "Unable to get chromEnd"
# We got none of these

# possible exceptions logged to snpLocType.exceptions:
# RefAlleleWrongSize
# this run got just 40 

# morph chrN_snpTmp 

mysql> desc chr1_snpTmp;

#  +---------------+-------------+------+-----+---------+-------+
#  | Field         | Type        | Null | Key | Default | Extra |
#  +---------------+-------------+------+-----+---------+-------+
#  | snp_id        | int(11)     | NO   |     |         |       |
#  | ctg_id        | int(11)     | NO   |     |         |       |
#  | chromStart    | int(11)     | NO   |     |         |       |
#  | chromEnd      | int(11)     | NO   |     |         |       |
#  | loc_type      | tinyint(4)  | NO   |     |         |       |
#  | orientation   | tinyint(4)  | NO   |     |         |       |
#  | allele        | blob        | YES  |     | NULL    |       |
#  +---------------+-------------+------+-----+---------+-------+

/cluster/home/heather/kent/src/hg/snp/snpLoad/snpLoctype dbSnpHumanBuild125 ref_haplotype

# expand allele as necessary
# report syntax errors to snpExpandAllele.errors
# this run had 63 of these
# possible exceptions logged to snpExpandAllele.exceptions:
# RefAlleleWrongSize
# this run has 512 

/cluster/home/heather/kent/src/hg/snp/snpLoad/snpExpandAllele dbSnpHumanBuild125 ref_haplotype

# the next few steps prepare for working in UCSC space
# sort by position
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpSort dbSnpHumanBuild125 ref_haplotype
# get hg17 nib files
# get hg17 chromInfo, load into dbSnpHumanBuild125 with editted path
hgsql -e "rename table chrMT_snpTmp to chrM_snpTmp" dbSnpHumanBuild125

# lookup reference allele in nibs
# keep reverse complement to use in error checking (snpCheckAlleles)
# check here for SNPs larger than 1024
# errAbort if detected
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpRefUCSC dbSnpHumanBuild125

# morph chrN_snpTmp 

mysql> desc chr1_snpTmp;

#  +--------------------+-------------+------+-----+---------+-------+
#  | Field              | Type        | Null | Key | Default | Extra |
#  +--------------------+-------------+------+-----+---------+-------+
#  | snp_id             | int(11)     | NO   |     |         |       |
#  | ctg_id             | int(11)     | NO   |     |         |       |
#  | chromStart         | int(11)     | NO   |     |         |       |
#  | chromEnd           | int(11)     | NO   |     |         |       |
#  | loc_type           | tinyint(4)  | NO   |     |         |       |
#  | orientation        | tinyint(4)  | NO   |     |         |       |
#  | allele             | blob        | YES  |     | NULL    |       |
#  | refUCSC            | blob        | YES  |     | NULL    |       |
#  | refUCSCReverseComp | blob        | YES  |     | NULL    |       |
#  +--------------------+-------------+------+-----+---------+-------+

# compare allele from dbSNP to refUCSC
# locType between is excluded from this check
# log exceptions to snpCheckAllele.exceptions
# if SNP is positive strand, expect allele == refUCSC
# log RefAlleleMismatch if not
# if SNP is negative strand, if not allele == refUCSC, then check for allele == refUCSCReverseComp
# If allele == refUCSCRevComp, log RefAlleleNotRevComp
# If allele doesn't match either of refUCSC or refUCSCReverseComp, log RefAlleleMismatch
# This run we got:
# 0 RefAlleleMismatch
# 49763   RefAlleleNotRevComp
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpCheckAlleles dbSnpHumanBuild125

# add class, observed and molType from chrN_snpFasta tables
# log errors to snpReadFasta.errors
# errors detected: no data available, duplicate data
# This run we got:
# 49 no data available
# 226048 duplicate
# chrN_snpFasta has class = 'in-del'
# we promote this to 'deletion' for locType 1&2 and to 'insertion' for locType 3

/cluster/home/heather/kent/src/hg/snp/snpLoad/snpReadFasta dbSnpHumanBuild125

# morph chrN_snpTmp
#  +--------------------+---------------+------+-----+---------+-------+
#  | Field              | Type          | Null | Key | Default | Extra |
#  +--------------------+---------------+------+-----+---------+-------+
#  | snp_id             | int(11)       | NO   |     |         |       |
#  | chromStart         | int(11)       | NO   |     |         |       |
#  | chromEnd           | int(11)       | NO   |     |         |       |
#  | loc_type           | tinyint(4)    | NO   |     |         |       |
#  | class              | varchar(255)  | NO   |     |         |       |
#  | orientation        | tinyint(4)    | NO   |     |         |       |
#  | molType            | varchar(255)  | NO   |     |         |       |
#  | allele             | blob          | YES  |     | NULL    |       |
#  | refUCSC            | blob          | YES  |     | NULL    |       |
#  | refUCSCReverseComp | blob          | YES  |     | NULL    |       |
#  | observed           | blob          | YES  |     | NULL    |       |
#  +--------------------+---------------+------+-----+---------+-------+

# generate exceptions for class and observed

# SingleClassBetweenLocType
# SingleClassRangeLocType
# NamedClassWrongLocType

# ObservedNotAvailable
# ObservedWrongFormat
# ObservedWrongSize
# ObservedMismatch

# RangeSubstitutionLocTypeExactMatch

# SingleClassTriAllelic
# SingleClassQuadAllelic

# This will also detect IUPAC symbols in allele

/cluster/home/heather/kent/src/hg/snp/snpLoad/snpCheckClassAndObserved dbSnpHumanBuild125

# add function
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpFunction dbSnpHumanBuild125

# add validation status and heterozygosity
# log error if validation status > 31 or missing
# this run we got 8 missing
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpSNP dbSnpHumanBuild125

# generate chrN_snp125 and snp125Exceptions tables
cp snpCheckAlleles.exceptions snpCheckAlleles.tab
cp snpCheckClassAndObserved.exceptions snpCheckClassAndObserved.tab
cp snpExpandAllele.exceptions snpExpandAllele.tab
cp snpLocType.exceptions snpLocType.tab
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpFinalTable dbSnpHumanBuild125

# PAR SNPs
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpPAR dbSnpHumanBuild125
hgsql -e 'load data local infile "snpPARexceptions.tab" into table snp125Exceptions' dbSnpHumanBuild125

# concat into snp125.tab
# cat chr*_snp125.tab >> snp125.tab
/bin/sh concat.sh

# load
hgsql dbSnpHumanBuild125 < /cluster/home/heather/kent/src/hg/lib/snp125.sql
hgsql -e 'load data local infile "snp125.tab" into table snp125' dbSnpHumanBuild125

# check for multiple alignments
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpMultiple dbSnpHumanBuild125
mysql> load data local infile 'snpMultiple.tab' into table snp125Exceptions;

# run and review snpCompareLoctype (currently tuned for 124/125 differences)
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpCompareLoctype dbSnpHumanBuild125 snp124subset snp125
# cat snpCompareLoctypeCounts.out
# exactToExact = 8310192
# exactToBetween = 107956
# exactToRange = 16200
# betweenToBetween = 206224
# betweenToExact = 4012
# betweenToRange = 715
# rangeToRange = 98648
# rangeToBetween = 3151
# rangeToExact = 6198
# oldToNew = 10224

# 12043 coord changes in exact (.1%)
# 1370 moved to different chroms

# 3664 coord changes in between (1.7%)
# 2260 off-by-one
# 13 moved to different chroms

# 22198 coord changes in range (22.5%)
# 19548 look like fixes: observedLengthOld != coordSpanOld && observedLengthNew == coordSpanNew
# 1296 look like errors: observedLengthOld == coordSpanOld && observedLengthNew != coordSpanNew

# load on hgwdev
cp snp125.tab /cluster/home/heather/transfer/snp
hgsql dbSnpHumanBuild125 -e 'select * from snp125Exceptions' > /cluster/home/heather/transfer/snp/snp125Exceptions.tab
ssh hgwdev
mysql> load data local infile 'snp125.tab' into table snp125; 

# create indexes
mysql> alter table snp125 add index name (name);
mysql> alter table snp125 add index chrom (chrom, bin);

mysql> load data local infile 'snp125Exceptions.tab' into table snp125Exceptions; 
mysql> alter table snp125Exceptions add index name(name);

# create snp125ExceptionDesc table
cd /cluster/data/dbSnp
# add counts to exception.template
hgsql hg17 < snp125ExceptionDesc.sql
mysql> load data local file 'exception.template' into table snp125ExceptionDesc;

#######
# Add new case for ObservedWrongSize (Heather June 9, 2006)
# revisions 1.25 and 1.26 kent/src/hg/snp/snpLoad/snpCheckClassAndObserved.c
ssh kkr5u00
cd /scratch/snp/125/human
/bin/csh pipeline.csh
# wait 35 minutes
grep ObservedWrongSize snpCheckClassAndObserved.exceptions > ObservedWrongSize
grep ObservedWrongSize snpPARexceptions.tab >> ObservedWrongSize
cp ObservedWrongSize /cluster/home/heather/transfer/snp
ssh hgwdev
hgsql -e 'alter table snp125Exceptions drop index name' hg17
hgsql -e 'load data local infile "/cluster/home/heather/transfer/snp/ObservedWrongSize" into table snp125Exceptions' hg17
hgsql -e 'alter table snp125Exceptions add index name'

# fix counts
hgsql -e 'select count(*), exception from snp125Exceptions group by exception' hg17

+----------+------------------------------------+
| count(*) | exception                          |
+----------+------------------------------------+
|   785903 | MultipleAlignments                 |
|      623 | NamedClassWrongLocType             |
|     7686 | ObservedMismatch                   |
|     4333 | ObservedNotAvailable               |
|       97 | ObservedWrongFormat                |
|    73558 | ObservedWrongSize                  |
|      466 | RangeSubstitutionLocTypeExactMatch |
|       62 | RefAlleleMismatch                  |
|    99849 | RefAlleleNotRevComp                |
|     1278 | RefAlleleWrongSize                 |
|    20749 | SingleClassBetweenLocType          |
|     2306 | SingleClassQuadAllelic             |
|    15639 | SingleClassRangeLocType            |
|    19330 | SingleClassTriAllelic              |
+----------+------------------------------------+

# edit /cluster/data/dbSNP/exception.template (need to automate this)
hgsql -e 'delete from snp125ExceptionDesc' hg17
hgsql -e 'load data local infile "/cluster/data/dbSNP/exception.template" into table snp125ExceptionDesc' hg17


###########################

# add rs_fasta to seq/extFile (Heather Nov 2006)
# use 126 rs_fasta files because I didn't save 125 version
ssh hgwdev
mkdir /gbdb/hg17/snp
ln -s /cluster/store12/snp/126/human/rs_fasta/snp.fa /gbdb/hg17/snp/snp.fa
cd /cluster/store12/snp/126/human/rs_fasta
hgLoadSeq hg17 /gbdb/hg18/snp/snp.fa
# clean up after hgLoadSeq
rm seq.tab

# look up id in extFile
# move into separate table
hgsql hg17 < snpSeq.sql
hgsql -e 'insert into snpSeq select acc, file_offset from seq where extFile = 33852294' hg17
hgsql -e 'delete from seq where extFile = 33852294' hg17
hgsql -e 'alter table snpSeq add index acc (acc)' hg17

#############################################################
# Get panTro2 and rheMac2 allele for all SNPs (Heather, Dec 2006, Feb 2007 and
# June 2007 [partial fix released 6/25/07: using hg17 instead of hg18 liftOver
# files... for most but not all chroms! :( not documented below; error found 
# by user]
# 1/11/08 (angie): re-running panTro2Qual and subsequent chimp & summary 
# steps, so hg17 liftOver files will have been used for all outputs.
# Deletions will probably lift okay
# The insertions have start == end so none of them will lift
# 1/24/08 (angie): constant quality score of 98 for chimp chr{21,M,Y,Y_random}
# was previously put in score field -- corrected to orthoScore.

ssh hgwdev
cd /san/sanvol1/snp/liftOver/hg17
mkdir panTro2All
mkdir rheMac2All
mkdir input
cd input
hgsql -N -e 'select chrom, chromStart, chromEnd, name, score, strand from snp125' hg17 > snp125.bed
lineFileSplit snp.bed lines 100000 snp-
ln -s /san/sanvol1/snp/liftOver/hg17/input /san/sanvol1/snp/liftOver/hg17/panTro2All/input
ln -s /san/sanvol1/snp/liftOver/hg17/input /san/sanvol1/snp/liftOver/hg17/rheMac2All/input
cd ../panTro2All
./makeJobList.csh
mkdir output
mkdir unmapped
cd ../rheMac2All
./makeJobList.csh
mkdir output
mkdir unmapped

# cluster run
ssh pk
cd /san/sanvol1/snp/liftOver/hg17/panTro2All
para create jobList; para try; para check; para push
para time
# Completed: 108 of 108 jobs
# CPU time in finished jobs:      67758s    1129.29m    18.82h    0.78d  0.002 y
# IO & Wait Time:                   961s      16.02m     0.27h    0.01d  0.000 y
# Average job time:                 636s      10.60m     0.18h    0.01d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:            1543s      25.72m     0.43h    0.02d
# Submission to last job:         61513s    1025.22m    17.09h    0.71d

cd /san/sanvol1/snp/liftOver/hg17/rheMac2All
para create jobList; para try; para check; para push
para time
# Completed: 108 of 108 jobs
# CPU time in finished jobs:       1833s      30.56m     0.51h    0.02d  0.000 y
# IO & Wait Time:                  1744s      29.06m     0.48h    0.02d  0.000 y
# Average job time:                  33s       0.55m     0.01h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              82s       1.37m     0.02h    0.00d
# Submission to last job:         59987s     999.78m    16.66h    0.69d

# add sequence
# next time do this at the same time as lift

cd /san/sanvol1/snp/liftOver/hg17
mkdir panTro2Seq
mkdir panTro2Seq/input
mkdir panTro2Seq/output
cp panTro2All/output/snp*out panTro2Seq/input
cd panTro2Seq
./makeJobList.csh

cat << 'EOF' > makeJobList.csh
#!/bin/tcsh
rm -f jobList
foreach fileName (`ls input/*`)
    set baseName = $fileName:t
    echo $baseName
    echo "/cluster/home/heather/kent/src/hg/snp/snpLoad/fetchSeq $fileName /scratch/hg/panTro2/panTro2.2bit output/$baseName" >> jobList
end
'EOF'

cd /san/sanvol1/snp/liftOver/hg17
mkdir rheMac2Seq
mkdir rheMac2Seq/input
mkdir rheMac2Seq/output
cp rheMac2All/output/snp*out rheMac2Seq/input
cd rheMac2Seq
cat << 'EOF' > makeJobList.csh
#!/bin/tcsh
rm -f jobList
foreach fileName (`ls input/*`)
    set baseName = $fileName:t
    echo $baseName
    echo "/cluster/home/heather/kent/src/hg/snp/snpLoad/fetchSeq $fileName /scratch/hg/rheMac2/rheMac2.2bit output/$baseName" >> jobList
end
'EOF'

# cluster run for sequence
ssh pk
cd /san/sanvol1/snp/liftOver/hg17/panTro2Seq
para create jobList; para try; para check; para push
para time
# Completed: 108 of 108 jobs
# CPU time in finished jobs:      30509s     508.48m     8.47h    0.35d  0.001 y
# IO & Wait Time:                   325s       5.42m     0.09h    0.00d  0.000 y
# Average job time:                 286s       4.76m     0.08h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:             551s       9.18m     0.15h    0.01d
# Submission to last job:          1195s      19.92m     0.33h    0.01d

cd /san/sanvol1/snp/liftOver/hg17/rheMac2Seq
para create jobList; para try; para check; para push
para time
# Completed: 108 of 108 jobs
# CPU time in finished jobs:      28517s     475.28m     7.92h    0.33d  0.001 y
# IO & Wait Time:                   576s       9.61m     0.16h    0.01d  0.000 y
# Average job time:                 269s       4.49m     0.07h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:             509s       8.48m     0.14h    0.01d
# Submission to last job:          1166s      19.43m     0.32h    0.01d

# quality scores 
# This takes about 24 hours for each species!!  Ugh.
# Solution is to use -bedFile argument to hgWiggle
ssh hgwdev
cd /san/sanvol1/snp/liftOver/hg17
cd panTro2Seq/output
cat << 'EOF' > concat.csh
#!/bin/tcsh
rm -f all.out
foreach fileName (`ls snp*.out`)
    cat $fileName >> all.out
end
'EOF'

sort all.out > all.sort
rm all.out

cd /san/sanvol1/snp/liftOver/hg17
mkdir panTro2Qual
cp panTro2Seq/output/all.sort panTro2Qual
cd panTro2Qual
mkdir input
splitFileByColumn all.sort input
mkdir output

# If we do this again, we should write a c program to read qac files into
# memory -- much faster than one hgWiggle process per line.
cat << 'EOF' > addQual.pl
#!/usr/bin/perl -W
$db=shift;
$chromName=shift;
while (<STDIN>)
{
    my @fields = split;
    my $chrom = $fields[0];
    my $chromStart = $fields[1];
    my $chromEnd = $fields[2];
    my $name = $fields[3];
    my $strand = $fields[5];
    my $allele = $fields[6];

    $cmd="hgWiggle -db=$db -chrom=$chromName -position=$chrom:$chromStart-$chromStart -rawDataOut quality";
    open(RESULT, "$cmd |") or die "can't start '$cmd'\n";
    while ($line = <RESULT>)
    {
        $score = int($line);
        print "$chrom\t$chromStart\t$chromEnd\t$name\t$score\t$strand\t$allele\n";
    }
}
'EOF'
cat << 'EOF' > getQual.csh
#!/bin/tcsh
foreach fileName (`ls input/*`)
    set chromName = $fileName:t:r
    echo $chromName
    addQual.pl panTro2 $chromName < $fileName > output/$chromName
end
'EOF'
# << emacs
./getQual.csh

cd rheMac2Seq/output
concat.csh
sort all.out > all.sort
rm all.out
cd /san/sanvol1/snp/liftOver/hg17
mkdir rheMac2Qual
cp rheMac2Seq/output/all.sort rheMac2Qual
cd rheMac2Qual
mkdir input
splitFileByColumn all.sort input
mkdir output
./getQual.csh

# concatenate, merge and load
# chimp has no qual scores for chr21, chrY and chrM, just use seq files
cd /san/sanvol1/snp/liftOver/hg17/panTro2Qual/output
grep chr21 ../../panTro2Seq/output/all.sort > chr21
grep chrY ../../panTro2Seq/output/all.sort | grep -v random > chrY
grep chrY ../../panTro2Seq/output/all.sort | grep random > chrY_random
grep chrM ../../panTro2Seq/output/all.sort > chrM
#-----------------------------------------------------------------------------
# 1/11/08: replace outputs for chroms that apparently were skipped in the June 
# run, and re-run subsequent steps for chimp.  
cd /san/sanvol1/snp/liftOver/hg17/panTro2Qual
mv output output-jun25
foreach f (output-jun25/chr*)
  if ( "X"`cmp $f output-feb26/$f:t` == "X" ) then
    echo $f:t
  endif
end
#chr21
#chrM
#chrY
#chrY_random
# <<-- those are the ones that may not have actually been regenerated.

# It appears that the Feb. outputs, instead of the June Seq files, were copied 
# to the June output for those chroms.  oops!
# As a minor improvement, skip duplicate rows instead of just copying.
foreach chr (chr21 chrM chrY chrY_random)
  echo $chr
  uniq input/$chr.sort > output/$chr
end
# << emacs

mkdir output-jun25-incorrect
mv output-jun25/chr{21,M,Y,Y_random} output-jun25-incorrect
cat output-jun25/chr* output/chr* > output/qual.tab

# end 1/11/08 fix-specific; proceeding to post-concat.csh chimp steps.
#-----------------------------------------------------------------------------
./concat.csh
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpOrtho hg17 snp125 qual.tab
hgLoadBed hg17 snp125OrthoPanTro2 snpOrtho.tab -tab -sqlTable=/cluster/home/heather/kent/src/hg/lib/snp125OrthoPanTro2.sql
#Loaded 9591230 elements of size 17
# previously 9590961
# add index
hgsql hg17
  alter table snp125OrthoPanTro2 add index name (name);
  alter table snp125OrthoPanTro2 add index chrom (chrom, bin);
# 1/24/08: these used to set score; should have set orthoScore all along.
# tweak to match panTro2 assembly
  update snp125OrthoPanTro2 set orthoScore = 98 where orthoChrom = "chr21";
#Query OK, 129170 rows affected (25.37 sec)
#Rows matched: 129170  Changed: 129170  Warnings: 0
  update snp125OrthoPanTro2 set orthoScore = 98 where orthoChrom = "chrY";
#Query OK, 22081 rows affected (25.16 sec)
#Rows matched: 22081  Changed: 22081  Warnings: 0
  update snp125OrthoPanTro2 set orthoScore = 98 where orthoChrom = "chrY_random";
#Query OK, 155 rows affected (25.41 sec)
#Rows matched: 155  Changed: 155  Warnings: 0

# macaque
cd /san/sanvol1/snp/liftOver/hg17/rheMac2Qual/output
./concat.csh
/cluster/home/heather/kent/src/hg/snp/snpLoad/snpOrtho hg17 snp125 qual.tab
hgLoadBed hg17 snp125OrthoRheMac2 snpOrtho.tab -tab -sqlTable=/cluster/home/heather/kent/src/hg/lib/snp125OrthoRheMac2.sql
# add index
alter table snp125OrthoRheMac2 add index name (name);
alter table snp125OrthoRheMac2 add index chrom (chrom, bin);

# get hapmap subset for chimp
# skip if lift wasn't size 1
# this run 124822 skipped
cd /cluster/data/hg17/bed/hapmap/rel21a
time /cluster/home/heather/kent/src/hg/snp/snpLoad/hapmapOrtho hg17 \
  hapmapSnpsCombined snp125OrthoPanTro2
#108.505u 16.869s 2:26.22 85.7%  0+0k 0+0io 4pf+0w
hgLoadBed hg17 hapmapAllelesChimp hapmapOrtho.tab -tab -sqlTable=/cluster/home/heather/kent/src/hg/lib/hapmapAllelesChimp.sql
#Loaded 3930564 elements of size 13
hgsql hg17 -e 'alter table hapmapAllelesChimp add index name(name); \
  alter table hapmapAllelesChimp add index chrom (chrom, bin);'

# get hapmap subset for macaque
# this run 106607 skipped
/cluster/home/heather/kent/src/hg/snp/snpLoad/hapmapOrtho hg17 hapmapSnpsCombined snp125OrthoRheMac2
hgLoadBed hg17 hapmapAllelesMacaque hapmapOrtho.tab -tab -sqlTable=/cluster/home/heather/kent/src/hg/lib/hapmapAllelesMacaque.sql
rm hapmapOrtho.tab
rm hapmapOrtho.err
rm bed.tab
alter table hapmapAllelesMacaque add index name(name);
alter table hapmapAllelesMacaque add index chrom (chrom, bin);

##############################################################################
# HapMap Recombination Rate Phase 2 (Heather Feb. 2006)
# Contacts:
# Gil McVean [mcvean@stats.ox.ac.uk]                                                                                                 
# Colin Freeman [cfreeman@stats.ox.ac.uk]                                                                                            
# Simon Myers [smyers@broad.mit.edu]         
# Data is missing chromEnd.  I am setting chromEnd = chromStart + 1 as a
# kludge for now.
# Solution is to interpolate range but remove gaps.

## ****************************************
# This is a bad assumption about the data format -- here is a description.
## ****************************************
# The recombination rates are for the regions _between_ snps, so these
# files need to be processed slightly differently.  For each line i in
# the file (except the header and the last line), the recombination
# rate is for the position on the current line minus 1 [pos(${i})-1] to
# the position on the subsequent line [pos({$i+1})].  The precision is
# a bit obnoxious and can be truncated to 3 or 4 significant figures.
# (Note that the recombination rate on the last line is 0, as this is a
# placeholder.)  Here is an example:
#
# > head genetic_map_chr1.txt 
# position COMBINED_rate(cM/Mb) Genetic_Map(cM)
# 45413 2.98182170902573 0
# 72434 2.08241435350679 0.0805718043995841
# 78032 2.08135840137317 0.0922291599505152
# 244859 2.88844902005393 0.439455937976397
# 604461 2.88749757426825 1.47814798248583
# 604484 2.88586385769306 1.47821439493004
# 605296 2.88389196108775 1.48055771638249
#
### BED format (like a bedGraph)
# chr1 45412  72434  2.982
# chr1 72433  78302  2.082
# chr1 78031  244859 2.081
# chr1 244858 604461 2.888
# chr1 604460 604484 2.887
# chr1 604483 605296 2.886
# chr1 605295 .....  2.884
#
# See /cluster/data/hg16/bed/hapmap/recombination/Perlegen/makeBed.pl for an example.  /cluster/data/hg16/bed/hapmap/recombination/Perlegen/cmds.csh is also useful. 
## ****************************************

ssh hgwdev
cd /cluster/data/hg17/bed/hapmap
mkdir recombination
cd recombination
mkdir phase2
cd phase2
wget --no-check-certificate -N https://mathgen.stats.ox.ac.uk/HapMap_Phase2_rates_hotspots/HapMap_Phase2_rates_hotspots.tgz  
# data also available at
# http://www.hapmap.org/downloads/recombination/2006-10_rel21_phaseII
gunzip *.tgz
tar xvf *.tar
cat << 'EOF' > makeBed.csh
#!/bin/tcsh
rm -f recomb.bed
foreach chrom (`cat chrom.list`)
    echo $chrom
    set fileName=`echo $chrom | awk '{printf "genetic_map_%s.txt", $1}'`
    makeBed.pl $chrom < $fileName >> recomb.bed
end
makeBed.pl chrX < genetic_map_chrX_par1.txt >> recomb.bed
makeBed.pl chrX < genetic_map_chrX_non-par.txt >> recomb.bed
makeBed.pl chrX < genetic_map_chrX_par2.txt >> recomb.bed
'EOF'
cat << 'EOF' > makeBed.pl
#!/usr/bin/env perl
$chromName = shift;
while (<STDIN>) {
    my @fields = split;
    # skip header
    if ($fields[0] eq "position") { next; }
    print $chromName;
    print "\t";
    print $fields[0];
    print "\t";
    print $fields[0] + 1;
    print "\t";
    my $val1000 = $fields[1] * 1000;
    my $valRound = int($val1000);
    my $newVal = $valRound / 1000.0;
    print $newVal;
    print "\n";
}
'EOF'
./makeBed.csh
hgLoadBed hg17 snpRecombRateHapmapPhase2 recomb.bed -tab -bedGraph=4
hgsql -e 'alter table snpRecombRateHapmapPhase add index chrom (chrom, bin)' hg17

############
# UPDATE hg17 knownToVisiGene (2006-04-05 galt)
# Create table that maps between known genes and visiGene database 
    # mapping to other species such as mouse, zebrafish, frog
    # requires visiGene probe track vgImageProbes be created first
    knownToVisiGene hg17 -fromProbePsl=vgImageProbes
#############################################################
# ADD A NEW TRACK GROUP (DONE, 6/3/06, Fan)
# Create a new track group, "phenDis".

    echo 'INSERT INTO grp (name, label, priority) VALUES ("phenDis", "Phenotype and Disease Associations", 2.5)' \
   | hgsql hg17

#############################################################
# hgMut - Human Mutation track - Belinda Giardine
# list of tables by show tables like 'hgMut%'
# summary of current load June 7, 2006
#table definitions for autoSql
autoSql hgMut.as hgMut -dbLink
#move bin in struct so works as bed 4+
#hgMut.sql: change INDEXes as needed, put in enums
#shrink mutId to 64 chars, plus acc to 48
#data files and details under ~giardine/humPhen/
cd humPhen/hgMutData/April2006/
cat hgMutHbVar.txt hgMutPah.txt hgMutBgmut.txt hgMutCftr.txt hgMutARdb.txt > hgMutUnsorted.txt
grep "^chr" hgMutUnsorted.txt | sort -k1,1 -k2,2n > hgMut.bed
#create tables
hgsql hg17 < ../../hgMut.sql
#loading
hgLoadBed hg17 hgMut hgMut.bed -noSort -oldTable -tab
#load small vocab control tables
hgsql hg17 < hgMutLink.sql
hgsql hg17 < hgMutAttrClass.sql
hgsql hg17 < hgMutAttrName.sql
hgsql hg17 < hgMutSrc.sql
#from hgsql hg17
load data local infile "hgMutExtLinkHbVar.txt" into table hgMutExtLink;
load data local infile "hgMutExtLinkARdb.txt" into table hgMutExtLink;
load data local infile "hgMutExtLinkBgmut.txt" into table hgMutExtLink;
load data local infile "hgMutExtLinkCFTR.txt" into table hgMutExtLink;
load data local infile "hgMutExtLinkPah.txt" into table hgMutExtLink;
load data local infile "hgMutExtLinkSP.txt" into table hgMutExtLink;
load data local infile "hgMutAttrHbVar2.txt" into table hgMutAttr;
load data local infile "hgMutAttrHbvarProt2.txt" into table hgMutAttr;
load data local infile "hgMutAttrARdb.txt" into table hgMutAttr;
load data local infile "hgMutAttrARdbProt.txt" into table hgMutAttr;
load data local infile "hgMutAliasHbVar.txt" into table hgMutAlias;
load data local infile "hgMutAliasARdb.txt" into table hgMutAlias;
load data local infile "hgMutAliasBgmut.txt" into table hgMutAlias;
load data local infile "hgMutAliasPah.txt" into table hgMutAlias;
load data local infile "hgMutExtLinkHbVarOmim.txt" into table hgMutExtLink;
load data local infile "hgMutAttrLink.txt" into table hgMutAttrLink;
load data local infile "hgMutAttrSP.txt" into table hgMutAttr;

#############################################################
# gv*  Belinda Giardine
# These tables are to replace the hgMut tables
# Most data is converted by me (on PSU machines) to loadable format and copied.
# The Swiss-Prot/UniProt data, is generated from the UniProt database at UCSC, 
# using perl scripts and table dumps.
# scripts in kent/src/hg/utils/gvParsers/swissProt/
# everything redone to not depend on the dv track in July 2006

#make list of variants from Swiss-Prot (make sure featureClass 23 is variant)
hgsql -N uniProt > spVars.txt <<end
select feature.acc, start, end-start, featureType.val, featureId.val from
feature, featureType, accToTaxon, featureId where featureClass=23 and
featureType=featureType.id and accToTaxon.acc=feature.acc and taxon=9606 and
feature.featureId=featureId.id;
end
#need list mapping 3 letter amino acids to 1 letter. (aminoInfoDump from PSU)
#known gene protein map (kgProtMap) has psl data from (blastp)
# with qName being the spId
hgsql -N hg17 > kgProtMapDump.txt <<end
select kgProtMap.* from kgProtMap, uniProt.feature where kgProtMap.qName =
uniProt.feature.acc;
end
#table join duplicates; perl script to throw out extra before use
uniqueRows < kgProtMapDump.txt > kgProtMapUniq.txt
#check variables for output and input file names
computeSpVars > errors.txt
#errors.txt will list variants that couldn't be mapped
#July 18, 2006
#37 gaps, 564 proteins (2228 variants) not in kgProtMap (test one did align)
#found 22389
#Swiss-Prot attributes: 
hgsql hg17 < listSPconnections.sql > listSPconnections.txt
hgsql proteome < listSpXref2.sql > listSpXref2.txt
convertOmimTitle > gvLinkSPomim.txt
hgsql hg17 < listGeneVals.sql > listGeneVals.txt
convertDisPoly > gvLinkSPuni.txt
cat gvLinkSPuni.txt gvLinkSPomim.txt > gvLinkSp.txt
cp gvLinkSp.txt ../../../gv/gvData/

#creating gv* tables and loading 
#June 27, 2006
autoSql gv.as gv -dbLink
#edit indexes and string lengths in .sql file
id=48, srcId=48, raKey=48, attrType=48,
primary key=index on bin and attr and link ids (id, attrType for attrs)
#do enums
#add unique index, to prevent doubles
UNIQUE KEY (chrom(12), chromStart, chromEnd, name)
#added id field to gvPos struct so can keep ID when change name
#    char *id;   /* Added field to hold ID if change name */
#set to null in gv.c file

#reload data July 2006 with more data and corrected Swiss-Prot data
#also moved gv*(except gvPos) and omimTitle to hgFixed
#prep data: concatenate all the gvPos data, sort
cat gvPosSP.txt gvPosHbVar.txt gvPosARdb.txt gvPosBgmut.txt gvPosCftr.txt
gvPosPah.txt gvPosSrd5a2.txt gvPosBrca.txt > gvPosAll.txt
grep "^chr" gvPosAll.txt | sort -k1,1 -k2,2n > gvPosSortedHg17.bed

#load tables
hgLoadBed hg17 gvPos gvPosSortedHg17.bed -noSort -oldTable -tab
hgsql hg17 < gvSrc.sql
hgsql hg17
        load data local infile "gvBrca.txt" into table gv;
        load data local infile "gvAttrBrca.txt" into table gvAttr;
        load data local infile "gvLinkBrca.txt" into table gvLink;
        load data local infile "gvLinkSP.txt" into table gvLink;
        load data local infile "gvLinkSPgene.txt" into table gvLink;
        load data local infile "gvSP.txt" into table gv;
        load data local infile "gvAttrSP.txt" into table gvAttr;
        load data local infile "gvAttrLongSP.txt" into table gvAttrLong;
        load data local infile "gvLinkHbVar.txt" into table gvLink;
        load data local infile "gvHbVar.txt" into table gv;
        load data local infile "gvAttrHbVar.txt" into table gvAttr;
        load data local infile "gvARdb.txt" into table gv;
        load data local infile "gvAttrARdb.txt" into table gvAttr;
        load data local infile "gvBgmut.txt" into table gv;
        load data local infile "gvAttrBgmut.txt" into table gvAttr;
        load data local infile "gvAttrLongBgmut.txt" into table gvAttrLong;
        load data local infile "gvLinkBgmut.txt" into table gvLink;
        load data local infile "gvCftr.txt" into table gv;
        load data local infile "gvAttrCftr.txt" into table gvAttr;
        load data local infile "gvPah.txt" into table gv;
        load data local infile "gvAttrPah.txt" into table gvAttr;
        load data local infile "gvLinkPah.txt" into table gvLink;
        load data local infile "gvSrd5a2.txt" into table gv;
        load data local infile "gvAttrSrd5a2.txt" into table gvAttr;
        load data local infile "gvAttrConservedDisease.txt" into table gvAttr;

#get disease association predictions for conserved variants
#get list a variants that are already done
hgsql -N hg17 > gvWithDiseaseStatus.txt <<end
select id from gvAttr where attrType = 'disease';
end
#use table browser to get variants that intersect most conserved track
#set conserved variants that are null to likely
computeDiseaseAssocCons > gvAttrConservedDisease.txt

#Belinda Giardine Sept 2006
#reload tables, removed ones with sequence mismatches, added label and strand
#added new lsbd BTKbase
#Sequence mismatches were determined by using the position in the reference
#sequence to fetch the sequence affected by the variant.  Then for substitions
#and deletions with the nts deleted listed, the sequence was compared.
#Insertions and large deletions could not be checked.

#Belinda Giardine Dec 2006
#reload tables, additions to previous sources and more IDbases
#details in hg18 doc

#Belinda Giardine Jan 2007
#reload tables, additions and corrections, details in hg18 doc

#############################################################
# Illumina Hap300 (Heather, July 2006)

ssh hgwdev
cd /cluster/data/hg17/bed
mkdir illumina
cd illumina
trim.pl < Illumina_HumanHap300_SNPlist_01.13.2006.txt > trim.out
hgsql hg17 < illuminaTmp.sql
hgsql -e "load data local infile 'trim.out' into table illuminaTmp" hg17
# illuminaLookup generates bin
/cluster/home/heather/kent/src/hg/snp/snpLoad/illuminaLookup hg17 illuminaTmp snp125 snp125Exceptions illuminaLookup.out illuminaLookup.err
# errors:
# unexpected chrom chr1 for snp rs1291584
# unexpected chrom chr17 for snp rs3826555
# unexpected locType between for snp rs2036773
# unexpected locType between for snp rs2249255
# unexpected locType between for snp rs8051412
# unexpected locType between for snp rs1017238
# unexpected locType between for snp rs5019493
# 16 with locType = range*
# 402 not found!
# None that have multiple alignments.
hgsql hg17 < snpArrayIllumina300.sql
hgsql -e "load data local infile 'illuminaLookup.out' into table snpArrayIllumina300" hg17
hgsql -e "alter table snpArrayIllumina300 add index name (name)" hg17
hgsql -e "alter table snpArrayIllumina300 add index chrom (chrom, bin)" hg17

#############################################################
# Illumina Hap550 and Hap650 (Heather, April 2007)
# Transfer from hg18 for Bert Gold at NCI

ssh hgwdev
cd /cluster/data/hg17/bed/illumina
hgsql hg18 < getHg18-550.sql > 550.hg18
hgsql hg18 < getHg18-650.sql > 650.hg18
# get name, chrom, chromStart, chromEnd, strand observed from snp125
# where class = "single" and locType = "exact" and chromEnd = chromStart + 1
# Including tri/quad allelic and multiple-aligning for now
hgsql hg17 < getHg17.sql > snp125single.hg17
# sort and join
sort 550.hg18 > 550.hg18.sort
sort 650.hg18 > 650.hg18.sort
sort snp125single.hg17 > snp125single.hg17.sort
# 560704 lines in 550.join
# 660137 lines in 650.join
# 687 lines in 550.missing
# 706 lines in 650.missing
join 550.hg18.sort snp125single.hg17.sort > 550.join
join 650.hg18.sort snp125single.hg17.sort > 650.join
join -v 1 550.hg18.sort snp125single.hg17.sort > 550.missing
join -v 1 650.hg18.sort snp125single.hg17.sort > 650.missing
# fix column order
awk '{print $2, $3, $4, $1, 0, $5, $6}' 550.join > 550.bed
awk '{print $2, $3, $4, $1, 0, $5, $6}' 650.join > 650.bed
# load
hgLoadBed hg17 snpArrayIllumina550 550.bed -sqlTable=snpArrayIllumina550.sql
hgLoadBed hg17 snpArrayIllumina650 650.bed -sqlTable=snpArrayIllumina650.sql
# indices
mysql> alter table snpArrayIllumina550 add index name (name);
mysql> alter table snpArrayIllumina550 add index chrom (chrom, bin);
mysql> alter table snpArrayIllumina650 add index name (name);
mysql> alter table snpArrayIllumina650 add index chrom (chrom, bin);

#############################################################
# Affy 500K (Heather, September 2006)
# look up rsId using position

ssh hgwdev
cd /cluster/data/hg17/bed/snp/affyData/500K
# awk to create bed format from tsv files
/bin/csh cmds.csh
hgsql hg17 < affy250Nsp.sql
hgsql hg17 < affy250Sty.sql
hgsql -e "load data local infile 'Mapping250K_Nsp.bed' into table affy250Nsp" hg17
hgsql -e "load data local infile 'Mapping250K_Sty.bed' into table affy250Sty" hg17

# look up dbSNP rsIDs using position
# affy250Nsp
# 4311 missing, 7276 multiple
/cluster/home/heather/kent/src/hg/snp/snpLoad/affyLookup hg17 affy250Nsp snp125
mv affyLookup.out affy250Nsp.bed
mv affyLookup.err affy250Nsp.err
hgsql hg17 < snpArrayAffy250Nsp.sql
hgLoadBed hg17 snpArrayAffy250Nsp affy250Nsp.bed -sqlTable=snpArray250Nsp.sql -tab
hgsql -e "alter table snpArrayAffy250Nsp add index name (name)" hg17
hgsql -e "alter table snpArrayAffy250Nsp add index chrom (chrom, bin)" hg17

# affy250Sty
# 3540 missing, 6901 multiple
/cluster/home/heather/kent/src/hg/snp/snpLoad/affyLookup hg17 affy250Sty snp125
mv affyLookup.out affy250Sty.bed
mv affyLookup.err affy250Sty.err
hgsql hg17 < snpArrayAffy250Sty.sql
hgLoadBed hg17 snpArrayAffy250Sty affy250Sty.bed -sqlTable=snpArray250Sty.sql -tab
hgsql -e "alter table snpArrayAffy250Sty add index name (name)" hg17
hgsql -e "alter table snpArrayAffy250Sty add index chrom (chrom, bin)" hg17


#############################################################
# Affy 10K (Sept. 2006, Heather)
# look up rsId using position

ssh hgwdev
cd /cluster/data/hg17/bed/snp/affyData/10K100Kagain

# affy10
# 14 missing, 807 multiple
cp affy10K.txt affy10Temp.bed
hgLoadBed hg17 affy10Temp affy10Temp.bed -sqlTable=affy10Temp.sql -tab -noBin
/cluster/home/heather/kent/src/hg/snp/snpLoad/affyLookup hg17 affy10Temp snp125
mv affyLookup.out affy10.bed
mv affyLookup.err affy10.err
hgLoadBed hg17 snpArrayAffy10 affy10.bed -sqlTable=snpArrayAffy10.sql -tab

# affy10v2
# 12 missing, 716 multiple
hgLoadBed hg17 affy10v2Temp affy10v2Temp.bed -sqlTable=affy10v2Temp.sql -tab -noBin
/cluster/home/heather/kent/src/hg/snp/snpLoad/affyLookup hg17 affy10v2Temp snp125
mv affyLookup.out affy10v2.bed
mv affyLookup.err affy10.errv2
hgLoadBed hg17 snpArrayAffy10v2 affy10v2.bed -sqlTable=snpArrayAffy10v2.sql -tab

# affy50HindIII
# 156 missing, 1396 multiple
hgLoadBed hg17 affy50HindIIITemp affy50HindIII.bed -sqlTable=affy50HindIIITemp.sql -tab -noBin
/cluster/home/heather/kent/src/hg/snp/snpLoad/affyLookup hg17 affy50HindIIITemp snp125
mv affyLookup.out affy50HindIII.bed
mv affyLookup.err affy50HindIII.err
hgLoadBed hg17 snpArrayAffy50HindIII affy50HindIII.bed -sqlTable=snpArrayAffy50HindIII.sql -tab
hgsql -e "alter table snpArrayAffy50HindIII add index name (name)" hg17
hgsql -e "alter table snpArrayAffy50HindIII add index chrom (chrom, bin)" hg17

# affy50XbaI
# 115 missing, 1745 multiple
hgLoadBed hg17 affy50XbaITemp affy50XbaI.bed -sqlTable=affy50XbaITemp.sql -tab -noBin
/cluster/home/heather/kent/src/hg/snp/snpLoad/affyLookup hg17 affy50XbaI snp125
mv affyLookup.out affy50XbaI.bed
mv affyLookup.err affy50XbaI.err
hgLoadBed hg17 snpArrayAffy50XbaI affy50XbaI.bed -sqlTable=snpArrayAffy50XbaI.sql -tab
hgsql -e "alter table snpArrayAffy50XbaI add index name (name)" hg17
hgsql -e "alter table snpArrayAffy50XbaI add index chrom (chrom, bin)" hg17


#########################################################################
# REGULATORY POTENTIAL (DONE - 2006-06-14 - Hiram)
    #	download data from "James Taylor" <james@bx.psu.edu>
    ssh kkstore02
    mkdir /cluster/store11/hg17/bed/regPotential7X
    cd /cluster/data/hg17/bed
    ln -s /cluster/store11/hg17/bed/regPotential7X ./regPotential7X
    cd regPotential7X
    
    #	This is a lot of data
    time for C in 1 2 3 4 5 6 7 8 9 X Y 10 11 12 13 14 15 16 17 18 19 20 21 22
    do
    wget --timestamping \
"http://www.bx.psu.edu/~james/esperr_rp_7way_scores/genome_scores_hg17/chr${C}.scores.truncated.bz2"
    done
    #	real    115m1.855s

    wget --timestamping \
"http://www.bx.psu.edu/~james/esperr_rp_7way_scores/genome_scores_hg17/trackDb.html" -O description.html

    time for C in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y
    do
	bzcat chr${C}.scores.truncated.bz2
    done | wigEncode -noOverlap  stdin regPotential7X.wig regPotential7X.wib
    #	Converted stdin, upper limit 1.00, lower limit -0.00
    #	real    33m48.487s

    #	Loading the table on hgwdev
    ssh hgwdev
    cd /cluster/data/hg17/bed/regPotential7X
    ln -s /cluster/data/hg17/bed/regPotential7X/regPotential7X.wib \
	/gbdb/hg17/wib/regPotential7X.wib
    #	using the tmpDir is faster since it is on local disk and it will
    #	clean up any temporary .tab file it creates there
    time hgLoadWiggle -tmpDir=/scratch/tmp \
	hg17 regPotential7X regPotential7X.wig

    #	How about a histogram of the data.
    ssh kolossus
    cd /cluster/data/hg17/bed/regPotential7X
    time hgWiggle -verbose=2 -doHistogram -hBinSize=0.01 -hBinCount=100 \
	-hMinVal=0.0 -db=hg17 regPotential7X > histogram.data 2>&1
    #	real    2m48.810s
    #	73 % of the data values are zero

    #	create download gzip files from the bz2 files:
    for F in chr*.scores.truncated.bz2
    do
	C=`echo $F | awk -F'.' '{print $1}'`
	echo -n "${C}.regPotential7X.hg17.gz working ... "
	bzcat ${F} | gzip > ${C}.regPotential7X.hg17.gz
	echo
    done

#########################################################################
####### RE-BUILD RGD HUMAN QTL TRACKS (DONE 06/21/06 Fan) ##############
# DELETED RECORD FROM rgdQtlLink SO CONSISTENT WITH REMOVAL FROM rgdQtl
# (DONE, 2006-06-30, hartera)

ssh hgwdev
mkdir -p /cluster/store8/rgd/human12062005
rm /cluster/data/hg17/bed/rgdQtl
ln -s /cluster/store8/rgd/human12062005 /cluster/data/hg17/bed/rgdQtl
cd /cluster/data/hg17/bed/rgdQtl

# download data files from RGD

wget --timestamp ftp://rgd.mcw.edu/pub/RGD_genome_annotations/human/rgd_human_qtl_12062005.gff

# remove extra line feed character at the end of lines

rmLf rgd_human_qtl_12062005.gff > rgdQtl.gff

# create rgdQtl.tab
awk '{print $1"\t"$4"\t"$5"\t"$10}'  rgdQtl.gff |sed -e 's/Chr/chr/g'| \
sed -e 's/"//g' |sed -e 's/RGD://g' | sed -e 's/;//g' |sort -u > rgdQtl.tab

# create rgdQtlLink.tab

cat rgdQtl.gff |cut -f 9 |sed -e 's/; Note /\t/g'|\
sed -e 's/Alignment //' |sed -e 's/;Note /\t/' |\
sed -e 's/"//g' |sed -e 's/RGD://' >j.tmp
cut -f 2 j.tmp >j.1
cut -f 1,3 j.tmp >j.2
paste j.1 j.2 |sort -u >rgdQtlLink.tab
rm j.1 j.2 j.tmp

# load rgdQtl table
hgLoadBed hg17 rgdQtl rgdQtl.tab

# check rgdQtl table
checkTableCoords hg17 rgdQtl

# Go the following error messages:

#hg17.rgdQtl has 2 records with end > chromSize.
#hg17.rgdQtl has 2 records with end > chromSize.
#hg17.rgdQtl has 2 records with end > chromSize.
#hg17.rgdQtl has 1 records with end > chromSize.
#hg17.rgdQtl has 2 records with end > chromSize.
#hg17.rgdQtl has 3 records with end > chromSize.
#hg17.rgdQtl has 1 records with end > chromSize.
#hg17.rgdQtl has 3 records with end > chromSize.
#hg17.rgdQtl has 2 records with end > chromSize.
#hg17.rgdQtl has 1 records with end > chromSize.
#hg17.rgdQtl has 3 records with end > chromSize.
#hg17.rgdQtl has 2 records with end > chromSize.
#hg17.rgdQtl has 1 records with end > chromSize.
#hg17.rgdQtl has 1 records with end > chromSize.
#hg17.rgdQtl has 1 records with end > chromSize.
#hg17.rgdQtl has 1 records with end > chromSize.
#hg17.rgdQtl has 1 records with end < start.

hgsql hg17 -N -e 'select "do1", name, c.size from rgdQtl r, chromInfo c where chromEnd > c.size and r.chrom=c.chrom' >doall

cat << '_EOF_' > do1
hgsql hg17 -e "update rgdQtl set chromEnd = '${2}' where name='${1}'"
'_EOF_'

chmod +x do*
doall

checkTableCoords hg17 rgdQtl
#hg17.rgdQtl has 1 records with end < start.

hgsql hg17 -e 'select * from rgdQtl where chromEnd < chromStart'
# bin     chrom   chromStart      chromEnd        name
# 9       chr10   7135612 371019  BW63_H
# 0       chr20   77628133        5242324 AASTH39_H

# Don't know why checkTableCoords only catches one of the two erros.
hgsql hg17 -e "update rgdQtl set chromStart = 271019 where name='BW63_H'"
hgsql hg17 -e "update rgdQtl set chromEnd = 7135612 where name='BW63_H'"

# Delete the following record.  The RGD QTL is very questionable. 
hgsql hg17 -e "delete from rgdQtl where name='AASTH39_H'"

# load rgdQtlLink table
hgsql hg17 -e "drop table hg17.rgdQtlLink;"
hgsql hg17 <~/kent/src/hg/lib/rgdQtlLink.sql
hgsql hg17 -e 'load data local infile "rgdQtlLink.tab" into table hg17.rgdQtlLink;'

# Delete the record from rgdQtlLink table that was removed from the rgdQtl
# table above. (hartera, 2006-06-30)
hgsql hg17 -e "delete from rgdQtlLink where name='AASTH39_H'"

########################################################################

#########################################################################
#Reload omimTitle table			Belinda Giardine June 28, 2006
#fetched omim.txt.Z from OMIM downloads.
#parse out title lines (*FIELD* TI)
convertTitle < omim.txt > omimTitle.txt
#load into omimTitle table
truncate table omimTitle;
load data local infile "omimTitle.txt" into table omimTitle;

#############################################################
# Lift SV track from hg16 (Heather, July 2006)
# hg16 SV track is comprised of 7 subtracks:
# cnpFosmid, cnpSebat, cnpIafrate, cnpSharp, delConrad, delMccarroll, delHinds
# Use the same table formats as hg16; pre-create
# (No bin for del tables)

cd /cluster/data/hg17/bed
mkdir svMixed
cd svMixed

# I got hg17 coords from Andy Sharp for cnpFosmid and delHinds
trimFosmid.pl < cnpFosmid.txt > cnpFosmid.bed
hgLoadBed -tab hg17 cnpFosmid cnpFosmid.bed
hinds.pl < hinds.txt > delHinds.bed
hgLoadBed -tab -noBin hg17 delHinds delHinds.bed

# (7-27-2006 Brooke Rhead -- edited the cnpFosmid table)
# According to Andy Sharp, the name='Gap' items should be removed from
# cnpFosmid.  I dumped the table, removed the 'Gap' lines, then dumped the
# table again.

cd /cluster/data/hg17/bed/svMixed
hgsql hg17 -e "select * from cnpFosmid" > cnpFosmid_withGaps.bed

hgsql hg17
delete from cnpFosmid where name='Gap';

hgsql hg17 -e "select * from cnpFosmid" > cnpFosmid_withoutGaps.bed

# Simple lifts for delMccarroll
cat << '_EOF_' > liftConrad.csh
#!/bin/csh
hgsql -N -e 'select * from delMccarroll' hg16 > delMccarroll.hg16
liftOver -minMatch=0.7 delMccarroll.hg16 /gbdb/hg16/liftOver/hg16ToHg17.over.chain.gz delMccarroll.bed delMccarroll.err
hgLoadBed -sqlTable=delMccarroll.sql -tab -noBin hg17 delMccarroll delMccarroll.bed
'_EOF_'

# Lift both chromStart/chromEnd and thickStart/thickEnd for delConrad and join
cat << '_EOF_' > liftConrad.csh
#!/bin/csh
hgsql -N -e 'select chrom, chromStart, chromEnd, name, score, strand from delConrad' hg16 > delConrad.hg16.1
hgsql -N -e 'select chrom, thickStart, thickEnd, name, score, strand from delConrad' hg16 > delConrad.hg16.2
liftOver -minMatch=0.7 delConrad.hg16.1 /gbdb/hg16/liftOver/hg16ToHg17.over.chain.gz delConrad.tmp.1 delConrad.err.1
liftOver -minMatch=0.7 delConrad.hg16.2 /gbdb/hg16/liftOver/hg16ToHg17.over.chain.gz delConrad.tmp.2 delConrad.err.2
trimConrad.pl < delConrad.tmp.1 > delConrad.trim.1
trimConrad.pl < delConrad.tmp.2 > delConrad.trim.2
sort delConrad.trim.1 > delConrad.sort.1
sort delConrad.trim.2 > delConrad.sort.2
join delConrad.sort.1 delConrad.sort.2 > delConrad.join
awk '{print $2, $3, $4, $1, 1000, $5, $7, $8}' delConrad.join > delConrad.bed
hgLoadBed -sqlTable=delConrad.sql -noBin hg17 delConrad delConrad.bed
'_EOF_'

# Andy Sharp says the Sebat data has already been lifted, so be conservative here
# Create hg16.cnpSebatLiftCandidate that excludes 5 rows that had wild proliferations
cat << '_EOF_' > liftSebat.csh
hgsql -N -e 'select chrom, chromStart, chromEnd, name, probes, individuals from cnpSebatLiftCandidate' hg16 > cnpSebat.hg16
liftOver -minMatch=0.7 -bedPlus=4 cnpSebat.hg16 /gbdb/hg16/liftOver/hg16ToHg17.over.chain.gz cnpSebat.bed cnpSebat.err
hgLoadBed -sqlTable=cnpSebat.sql -tab hg17 cnpSebat cnpSebat.bed
'_EOF_'

# For Andy's data, use bacEndPairs first, then lift the remainder
cat << '_EOF_' > liftSharp.csh
# assumes a copy of hg16.cnpSharp in hg17.cnpSharpHg16Copy
/cluster/home/heather/kent/src/hg/snp/snpLoad/cnpLookup hg17 bacEndPairs cnpSharpHg16Copy cnpSharpLookup.out cnpSharpLookup.lift cnpSharpLookup.log
sed -e 's/Gain and Loss/GainAndLoss/' cnpSharpLookup.lift > cnpSharpLookup.lift.fix
mv cnpSharpLookup.lift.fix cnpSharpLookup.lift
liftOver -minMatch=0.7 -bedPlus=4 cnpSharpLookup.lift /gbdb/hg16/liftOver/hg16ToHg17.over.chain.gz cnpSharp.bed cnpSharp.err
sed -e 's/GainAndLoss/Gain And Loss/' cnpSharp.bed > cnpSharp.bed.fix
mv cnpSharp.bed.fix cnpSharp.bed
hgLoadBed -tab -sqlTable=cnpSharp.sql hg17 cnpSharp cnpSharpLookup.out
hgLoadBed -tab -oldTable hg17 cnpSharp cnpSharp.bed
'_EOF_'

# For the Iafrate data, the BAC End lookup wasn't good, so just lift
# Create hg16.cnpIafrateLiftCandidate that excludes 2 rows that had wild proliferations
cat << '_EOF_' > liftIafrate.csh
hgsql -N -e 'select chrom, chromStart, chromEnd, name, variationType, score from cnpIafrateLiftCandidate' hg16 > cnpIafrate.hg16
sed -e 's/Gain and Loss/GainAndLoss/' cnpIafrate.hg16 > cnpIafrate.hg16.fix
mv cnpIafrate.hg16.fix cnpIafrate.hg16
liftOver -minMatch=0.7 -bedPlus=4 cnpIafrate.hg16 /gbdb/hg16/liftOver/hg16ToHg17.over.chain.gz cnpIafrate.bed cnpIafrate.err
sed -e 's/GainAndLoss/Gain And Loss/' cnpIafrate.bed > cnpIafrate.bed.fix
mv cnpIafrate.bed.fix cnpIafrate.bed
hgLoadBed -sqlTable=cnpIafrate.sql -tab hg17 cnpIafrate cnpIafrate.bed
'_EOF_'

##############################################################################
# Add HapMap CNVRs from Matt Hurles (Heather Dec 2006)
ssh hgwdev
cd /cluster/data/hg17/bed/svRedon
# File from Matthew Hurles (meh@sanger.ac.uk) was essentially bed 4
# I decided to use bed 6 with score always 0 and strand always +
awk '{printf "%st%d\t%d\tcnp%s\t0\t%s\n", $1, $4, $5, $3, $7}' input.gff > input.bed
hgLoadBed hg17 cnpRedon input.bed

##############################################################################
#  dbRIP POLYALUL1SVA track added (2006-07-14 - DONE - Hiram)
#  dbRIP polyAluL1SVA
#	Data provider: Dr. Liang at the Liang lab:
#	http://falcon.roswellpark.org/index.html
#	Ping.Liang@roswellpark.org

#	Adding this track is a new data type into our browser.
#	data definitions for dbRIP and polyGenotype were added to
#	the hg/lib/ directory:
#	-rw-rw-r--  1  351 Jul 13 12:20 polyGenotype.as
#	-rw-rw-r--  1  694 Jul 13 12:22 polyGenotype.sql
#	-rw-rw-r--  1 6398 Jul 13 12:22 polyGenotype.c
#	-rw-rw-r--  1   980 Jul 10 17:59 dbRIP.as
#	-rw-rw-r--  1 11408 Jul 13 11:16 dbRIP.c
#	-rw-rw-r--  1  1578 Jul 13 12:06 dbRIP.sql
#	With associated .h files in hg/inc/
#	-rw-rw-r--  1 4600 Jul 10 18:00 dbRIP.h
#	-rw-rw-r--  1 4375 Jul 13 16:16 polyGenotype.h
#	Changes in hgTracks and hgc to make this track appear as it does
#	at their browser:
#	http://falcon.roswellpark.org:9090/cgi-bin/hgTables

#	For this first instance of the track, the data was obtained
#	directly from their Genome browser via the tables browser,
#	dumping the tables:
#	hg17.polyAluL1 and hg17.polyGenotype
#	saving these data dumps to:
#		(after a couple of versions were used ...)
    ssh hgwdev
    mkdir /cluster/data/hg17/bed/dbRIP
    cd /cluster/data/hg17/bed/dbRIP
    #	-rw-rw-r--  1  994485 Aug  1 16:03 dbRIP.2006-08-01.txt.gz
    #	-rw-rw-r--  1   18532 Aug  1 16:05 polyGenotype.2006-08-01.txt.gz

    #	Rearrange their data columns to more closely match the
    #	standard BED definitions, and split into three different
    #	data sets:
zcat dbRIP.2006-08-01.txt.gz | headRest 1 stdin | awk -F'\t' '
{
chromStart=$6
chromStart -= 1
chromEnd=$7
if (match($1,"^RIP_SVA_.*")) {
printf "%s\t%s\t%s\t%s\t0\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $5, chromStart, chromEnd, $1, $12, $2, $3, $4, $8, $9, $10, $11, $15, $13, $14, $16, $17, $18, $19, $20, $21
}
}' | sort -k1,1 -k2,2n > dbRIP.SVA.txt

zcat dbRIP.2006-08-01.txt.gz | headRest 1 stdin | awk -F'\t' '
{
chromStart=$6
chromStart -= 1
chromEnd=$7
if (match($1,"^RIP_L1_.*")) {
printf "%s\t%s\t%s\t%s\t0\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $5, chromStart, chromEnd, $1, $12, $2, $3, $4, $8, $9, $10, $11, $15, $13, $14, $16, $17, $18, $19, $20, $21
}
}' | sort -k1,1 -k2,2n > dbRIP.L1.txt

zcat dbRIP.2006-08-01.txt.gz | headRest 1 stdin | awk -F'\t' '
{
chromStart=$6
chromStart -= 1
chromEnd=$7
if (match($1,"^RIP_Alu_.*")) {
printf "%s\t%s\t%s\t%s\t0\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $5, chromStart, chromEnd, $1, $12, $2, $3, $4, $8, $9, $10, $11, $15, $13, $14, $16, $17, $18, $19, $20, $21
}
}' | sort -k1,1 -k2,2n > dbRIP.Alu.txt

    #	Create three specific sql table create definitions:
    sed -e "s/dbRIP/dbRIP_SVA/" $HOME/kent/src/hg/lib/dbRIP.sql > dbRIP_SVA.sql
    sed -e "s/dbRIP/dbRIP_L1/" $HOME/kent/src/hg/lib/dbRIP.sql > dbRIP_L1.sql
    sed -e "s/dbRIP/dbRIP_Alu/" $HOME/kent/src/hg/lib/dbRIP.sql > dbRIP_Alu.sql

    #	And loading those three data tables:
    hgLoadBed -verbose=3 -maxChromNameLength=6 -strict -tab -notItemRgb \
        -sqlTable=dbRIP_SVA.sql hg17 dbRIP_SVA dbRIP.SVA.txt

    hgLoadBed -verbose=3 -maxChromNameLength=6 -strict -tab -notItemRgb \
	-sqlTable=dbRIP_L1.sql hg17 dbRIP_L1 dbRIP.L1.txt

    hgLoadBed -verbose=3 -maxChromNameLength=6 -strict -tab -notItemRgb \
	-sqlTable=dbRIP_Alu.sql hg17 dbRIP_Alu dbRIP.Alu.txt

    #	And an associated table of genotype frequencies
    #	Add three extra rows to the original data to provide a better handle
    #	on MySQL lookups for allele Frequency
    hgsql hg17 -e "drop table polyGenotype;"
    hgsql hg17 < $HOME/kent/src/hg/lib/polyGenotype.sql
    zcat polyGenotype.2006-08-01.txt.gz | headRest 1 stdin | \
awk -F'\t' '
{
sampleSize = $3 + $4 + $5
plus = ($3 * 2) + $4
minus = ($5 * 2) + $4
if ((plus + minus) < 1) { alleleFreq=0 } else
        { alleleFreq = plus / (plus + minus) }
if (sampleSize > 0) {
heteroZyg = (2 * alleleFreq * (1.0 - alleleFreq)) * ((sampleSize * 2)/((sampleSize * 2) - 1))
} else {
heteroZyg = 2 * alleleFreq * (1.0 - alleleFreq)
}
printf "%s\t%s\t%d\t%d\t%d\t%.3f\t%.3f\t%.3f\n", $1, $2, $3, $4, $5, sampleSize, alleleFreq, heteroZyg
}
' > polyGenotype.txt

    hgsql hg17 -e \
	'load data local infile "polyGenotype.txt" into table polyGenotype;'

    #	A composite track was added to human/hg17/trackDb.ra to contain
    #	these three tracks, and search methods to get the name column
    #	participating in the search.  Need to figure out how to get some
    #	of the other text-rich columns participating in the search.

##############################################################################
# hg17 -> hg15 LIFTOVER CHAINS (DONE 7/27/06 Fan)
    # I used a size of 10kb instead of 3kb for the split (blat query) sizes in
    # hg15.  This had a huge affect on the amount of hits in the blat, which
    # then had a huge effect on the amount of chains.  I should also mention 
    # that hg15 chromosomes chr1 and chr2 were split further 
    # into more than a single query file.  This helped a LOT in avoiding 
    # cluster hippos classically associated with those chroms.
    
    ######## LIFTOVER PREPARATION
    # Split up hg15
    ssh pk
    cd /san/sanVol1/scratch/hg15
    mkdir -p liftSplits/{split,lift}
    bash
    for fa in /cluster/data/hg15/?{,?}/*.fa; do
      c=`basename $fa .fa`
      echo $c
      faSplit -lift=liftSplits/lift/${c}.lft size $fa -oneFile 10000 liftSplits/split/$c
    done
    mkdir -p biggerSplits/split
    cd biggerSplits/
    ln -s ../liftSplits/lift
    cd split/
    ln -s ../../liftSplits/split/* .
    faSplit sequence chr1.fa 5 chr1_
    faSplit sequence chr2.fa 5 chr2_
    rm chr{1,2}.fa

    # Make some dirs
    # cd /san/sanVol1/scratch
    # mkdir -p hg17

    # Copy 11.ooc files to hg17 subdirectory.
    # cp -p /cluster/store5/gs.16/build33/11.ooc hg17

    ## First, copy over scripts. (Already done before)

    # mkdir -p /san/sanVol1/scratch/fan
    # cp -p /san/sanVol1/scratch/fan/*.sh /san/sanVol1/scratch/fan
    # cp /san/sanVol1/scratch/andy/psl.header /san/sanVol1/scratch/fan

    ######## LIFTOVER BLATING  

    # HG17
    ssh kk
    cd /cluster/data/hg17
    #makeLoChain-align hg17 /scratch/hg/hg17/nib hg15 /san/sanVol1/scratch/hg15/biggerSplits/split

    makeLoChain-align hg17 /scratch/hg/hg17/bothMaskedNibs hg15  /san/sanVol1/scratch/hg15/liftOver/biggerSplits/split 

# Completed: 2392 of 2392 jobs
# CPU time in finished jobs:   25651277s  427521.28m  7125.35h  296.89d  0.813 y
# IO & Wait Time:                 74118s    1235.30m    20.59h    0.86d  0.002 y
# Average job time:               10755s     179.25m     2.99h    0.12d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:           82545s    1375.75m    22.93h    0.96d
# Submission to last job:         82579s    1376.32m    22.94h    0.96d

    ssh kkstore02

    cd /cluster/data/hg17
    cd bed

    mv blat.hg15.2006-07-25 /san/sanVol1/scratch/hg17 

    ssh pk
    cd /san/sanVol1/scratch/hg17/blat.hg15.2006-07-25/run/
    sed 's/^blat/blat.sh/; s/\}.*$/}/' spec | awk '{print "/san/sanVol1/scratch/fan/" $0 " hg17ToHg15"}' > newspec
    para create newspec
    para try
    para push
# Completed: 2392 of 2392 jobs
# CPU time in finished jobs:     612316s   10205.26m   170.09h    7.09d  0.019 y
# IO & Wait Time:                 12421s     207.02m     3.45h    0.14d  0.000 y
# Average job time:                 261s       4.35m     0.07h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:            3524s      58.73m     0.98h    0.04d
# Submission to last job:          3588s      59.80m     1.00h    0.04d

    ######## LIFTOVER CHAINING
    # LIFTING
    ssh pk
    cd /san/sanVol1/scratch/fan
    cp mm7SplitLift.sh hg15SplitLift.sh

    # change andy to fan, mm7 to hg15, and chrX to chr2, and remove chrUn_random 
    vi hg15SplitLift.sh

    cat << 'EOF' > hg15ChainMergeSplit.sh
#!/bin/bash
cp -r chainRaw/ /scratch/fan/hg15Lifts
pushd /scratch/fan/hg15Lifts
mkdir chain
/cluster/bin/x86_64/chainMergeSort chainRaw/*.chain | /cluster/bin/x86_64/chainSplit chain stdin
cp -r chain `dirs +1`
rm -rf chain chainRaw
'EOF'

    chmod +x hg15ChainMergeSplit.sh

    # HG17
    cd /san/sanVol1/scratch/hg17/blat.hg15.2006-07-25/raw
    /san/sanVol1/scratch/fan/hg15SplitLift.sh
# There was an extra file, nib22.fa, under /cluster/data/hg15/nib, which should not be there.
# -rw-rw-r--  1 2429 protein  50466533 May 20  2003 nib22.fa
# This caused hg15SplitLift.sh to end abnormally.

    cd ../    
    mkdir chainRun chainRaw
    cd chainRun
    cat > gsub << 'EOF'
#LOOP
/cluster/bin/x86_64/axtChain -verbose=0 -linearGap=medium -psl $(path1) /scratch/hg/hg17/bothMaskedNibs  /san/sanVol1/scratch/hg15/nib {check out line+ ../chainRaw/$(root1).chain}
#ENDLOOP
'EOF'
    ls -1S ../psl/*.psl > in.lst
    gensub2 in.lst single gsub spec
    para create spec
    para try
    para push
    para time
# Completed: 44 of 44 jobs
# CPU time in finished jobs:       3596s      59.94m     1.00h    0.04d  0.000 y
# IO & Wait Time:                   919s      15.31m     0.26h    0.01d  0.000 y
# Average job time:                 103s       1.71m     0.03h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:             274s       4.57m     0.08h    0.00d
# Submission to last job:           284s       4.73m     0.08h    0.00d
    ######### CHAINMERGE/NET/NETSUBSET
    ssh kolossus
    mkdir -p /scratch/fan/hg15Lifts
    cd /scratch/fan/hg15Lifts

    cp -r /san/sanvol1/scratch/hg17/blat.hg15.2006-07-25/chainRaw/ .
    mkdir chain
    /cluster/bin/x86_64/chainMergeSort chainRaw/* | /cluster/bin/x86_64/chainSplit chain stdin
# about 30 minutes.

    cp -rp chain /san/sanvol1/scratch/hg17/blat.hg15.2006-07-25/
    rm -rf chain
    rm -rf chainRaw

    ssh pk
    cd /san/sanvol1/scratch/fan
    cat << 'EOF' > netOver.sh 
#!/bin/bash

chain=$1
chrom=`basename $chain .chain`
sizesHGOld=$2
sizesHG15=/cluster/data/hg15/chrom.sizes
chainDir=`dirname $chain`
blatDir=`dirname $chainDir`
net=${blatDir}/net/${chrom}.net
over=${blatDir}/over/${chrom}.over

mkdir -p ${blatDir}/{over,net}
/cluster/bin/x86_64/chainNet $chain $sizesHGOld $sizesHG15 $net /dev/null
/cluster/bin/x86_64/netChainSubset $net $chain $over
'EOF'
    chmod +x netOver.sh

    mkdir netRun

    cd netRun/

    find /san/sanVol1/scratch/hg17/blat.hg15.2006-07-25/chain -name "*.chain" \
     | awk '{print "/san/sanVol1/scratch/fan/netOver.sh " $1 " /cluster/data/hg17/chrom.sizes"}' > spec
    para create spec
    para push
    para time
# Completed: 46 of 46 jobs
# CPU time in finished jobs:        438s       7.30m     0.12h    0.01d  0.000 y
# IO & Wait Time:                   118s       1.97m     0.03h    0.00d  0.000 y
# Average job time:                  12s       0.20m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              28s       0.47m     0.01h    0.00d
# Submission to last job:            67s       1.12m     0.02h    0.00d

    ########## FINISHING
    ssh hgwdev

    # HG17
    cd /san/sanvol1/scratch/hg17/blat.hg15.2006-07-25/over
    cat * >> ../hg17ToHg15.over.chain
    cd ../
    rm -rf psl/ net/ chain/ chainRaw/ over/
    cd ../
    cp -rp blat.hg15.2006-07-25/ /cluster/data/hg17/bed

    cd /cluster/data/hg17/bed
    ln -s blat.hg15.2006-07-25 blat.hg15
    ln -s `pwd`/blat.hg15/hg17ToHg15.over.chain liftOver/hg17ToHg15.over.chain
    ln -s `pwd`/liftOver/hg17ToHg15.over.chain /gbdb/hg17/liftOver/hg17ToHg15.over.chain
    mkdir -p /usr/local/apache/htdocs/goldenPath/hg17/liftOver

    gzip /cluster/store5/gs.18/build35/bed/blat.hg15/hg17ToHg15.over.chain
    ln -s /cluster/store5/gs.18/build35/bed/blat.hg15/hg17ToHg15.over.chain.gz /gbdb/hg17/liftOver/
    cp -p /cluster/store5/gs.18/build35/bed/blat.hg15/hg17ToHg15.over.chain.gz /cluster/data/hg17/bed/liftOver 

    cd /usr/local/apache/htdocs/goldenPath/hg17/liftOver
    ln -s /cluster/store5/gs.18/build35/bed/blat.hg15/hg17ToHg15.over.chain.gz hg17ToHg15.over.chain.gz
    hgAddLiftOverChain hg17 hg15 

############################################################################

############################################################################
# Reload CCDS (ccdsInfo, ccdsGene, ccdsKgMap) (2006-08-15 markd)
    cd /cluster/data/genbank/data/ccds/hg17
    ftp ftp-private.ncbi.nih.gov (user ccds, needs password)
    ftp> get CCDS.20060815.tar.gz
    mkdir /scratch/tmp/ccds
    cd /scratch/tmp/ccds
    tar -zxf  /cluster/data/genbank/data/ccds/hg17/CCDS.20060815.tar.gz

    # import ccds database tables
    hgsql -e 'create database ccds'
    hgsql ccds </cluster/data/genbank/etc/createTables.sql 
    hgsql ccds </cluster/data/genbank/etc/createKeys.sql 
    /cluster/data/genbank/bin/x86_64/ccdsImport ccds data/*.txt

    # create and load ccdsGene and ccdsInfo tables from imported database
    /cluster/data/genbank/bin/x86_64/ccdsMkTables -loadDb ccds hg17 ccdsInfo ccdsGene
    checkTableCoords hg17 -verbose=2 ccdsGene

    /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap  -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap
    rm -rf /scratch/tmp/ccds

    # 2006-08-23 - found bug with some source genes missing from ccdsInfo, fixed ccdsMkTables
    # and reload
    /cluster/data/genbank/bin/x86_64/ccdsMkTables -loadDb ccds hg17 ccdsInfo ccdsGene
    checkTableCoords hg17 -verbose=2 ccdsGene
    joinerCheck -database=hg17 -identifier=ccdsGeneId ~/compbio/kent/src/hg/makeDb/schema/all.joiner
    /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap  -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap
    # request push of 
        ccdsGene
        ccdsInfo
        ccdsKgMap

    # << emacs
##########################################################################
#  hars 1 to 202  Sol 09/10/2006

    set bedDir = /gbdb/hg17/haseq/bed
    mkdir -p $bedDir/hars
    pushd /projects/hg/wet/Sol/hars1to49
    cp -p hars_1to202.hg17.bed  $bedDir/hars/hars_1to202.bed
    hgLoadBed hg17 hars         $bedDir/hars/hars_1to202.bed
    rm -f                       $bedDir/hars/hars_1to202.bed
    popd

# BUILD HPRD DATA FOR KNOWN GENE DETAILS PAGE LINKS (DONE 9/11/06)

# First, build hprdToCdna.tab and hprdToUniProt.tab. 
# See hg18.txt for details.

    cd ~/data/hprd
    mkdir hg17
    cd hg17

    hgsql hg17 -e 'drop table hprdToCdna'
    hgsql hg17 <~/src/hg/lib/hprdToCdna.sql
    hgsql hg17 -e 'load data local infile "../hprdToCdna.tab" into table hprdToCdna'

    hgsql hg17 -e 'drop table hprdToUniProt'
    hgsql hg17 <~/src/hg/lib/hprdToUniProt.sql
    hgsql hg17 -e 'load data local infile "../hprdToUniProt.tab" into table hprdToUniProt'

# build knownToHprd table

    hgsql hg17 -N -e 'select kgId,hprdId from hprdToCdna, kgXref where cdnaId=kgId' >j.kg1
    hgsql hg17 -N -e 'select kgId,hprdId from hprdToUniProt, kgXref where uniProtId=spId' >j.kg2

    cat j.kg1 j.kg2 |sort -u >knownToHprd.tab
    wc knownToHprd.tab

    hgsql hg17 -e 'drop table knownToHprd'
    hgsql hg17 <~/src/hg/lib/knownToHprd.sql

    hgsql hg17 -e 'load data local infile "knownToHprd.tab" into table knownToHprd'
    hgsql hg17 -e 'select count(*) from knownToHprd'

# 19,345 records created.

# remove temporary files.

    rm j*

# Do the same for hg17.  See hg17.txt for details.


############################################################################
# Reload CCDS (ccdsInfo, ccdsGene, ccdsKgMap) (2006-09-20 markd)
# Reloaded due to bug that results in multiple versions of the same accession
# in the ccdsInfo table.

    cd /cluster/data/genbank/data/ccds/hg17
    ftp ftp-private.ncbi.nih.gov (user ccds, needs password)
    get CCDS.20060920.tar.gz
    mkdir /scratch/tmp/ccds
    cd /scratch/tmp/ccds
    tar -zxf  /cluster/data/genbank/data/ccds/hg17/CCDS.20060920.tar.gz

    # import ccds database tables
    hgsql -e 'drop database ccds; create database ccds'
    hgsql ccds </cluster/data/genbank/etc/createTables.sql 
    hgsql ccds </cluster/data/genbank/etc/createKeys.sql 
    /cluster/data/genbank/bin/x86_64/ccdsImport ccds data/*.txt

    # create and load ccdsGene and ccdsInfo tables from imported database
    /cluster/data/genbank/bin/x86_64/ccdsMkTables -loadDb ccds hg17 ccdsInfo ccdsGene
    /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap  -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap
    checkTableCoords hg17 -verbose=2 ccdsGene
    joinerCheck -database=hg17 -identifier=ccdsGeneId ~/compbio/kent/src/hg/makeDb/schema/all.joiner
    rm -rf /scratch/tmp/ccds

    # request push of 
        ccdsGene
        ccdsInfo
        ccdsKgMap

    # << emacs


#########################################################
# BUILD GAD TRACK (Done, 9/26/06, Fan)

   mkdir /cluster/store/gad060926
   ln -s /cluster/store/gad060926 /cluster/data/gad

# Receive "allxlsAStxt.txt" from GAD/NIA 
# contact person: Shenoy, Narmada, shenoyn@grc.nia.nih.gov  
   
   hgsql hg17 -e 'drop table gadAll'
   hgsql hg17 <~/src/hg/lib/gadAll.sql
   hgsql hg17 -e 'load data local infile "allxlsAStxt.txt" into table gadAll ignore 2 lines'
   hgsql hg17 -e 'create index geneSymbol on gadAll(geneSymbol(10))'

# create gad table

   hgsql hg17 -N -e \
   'select "chr",chromosome, chromStart, chromEnd, geneSymbol from gadAll where chromStart <>0'|\
   sed -e 's/chr\t/chr/' |grep -v "chr\." |grep -v " "|sort -u >gad.bed

   hgLoadBed hg17 gad gad.bed

#####################################################################
# YALE TRANSCRIPTIONALLY ACTIVE REGIONS (TARs/TransFrags) TRACK IDENTIFIED 
# USING A WHOLE GENOME TILING ARRAY (DONE, 2006-10-20, hartera)
# Data is from the paper: Bertone et al. Science 24 December 2004:
# Vol. 306. no. 5705, pp. 2242 - 2246. From Mark Gerstein's lab at Yale.
# Contact at Yale: Joel S. Rozowsky, joel.rozowsky@yale.edu
# The data consist of Transcriptionally Active Regions (TARs or TransFrags)
# found using Affymetrix genome tiling arrays. The data is from the lab
# of Mark Gerstein at Yale.
     ssh kkstore02
     mkdir /cluster/data/hg17/bed/yaleBertoneTars/
     cd /cluster/data/hg17/bed/yaleBertoneTars/
     # download Bertone et al. data from this URL:
    #http://dart.gersteinlab.org/cgi-bin/ar/download.cgi?ID=TAR_data_NCBI31.txt
     # and put it in this directory.
     # The sequences used to design the microarrays were from
     # UCSC hg13/NCBI Build 31 so the sequences
     # should be aligned again using Blat since this is probably better
     # than using liftOver across so many assemblies.

     # Get sequences from TARs file and put in FASTA format:
     # Remove characters from Windows:
     dos2unix TAR_data_NCBI31.txt
     # The TARs are in order of IDs in the file so the first TAR has ID 1, the
     # second is 2 up to the last which is 17517. These IDs are used to link
     # to the DART database of TARs at Yale so use these IDs in the FASTA
     # header lines. Need to add "TAR" as prefix to ID so that it is unique
     # in the seq table.
   awk 'BEGIN {FS="\t";n=0;}{if ($1 ~ /^chr/) print ">TAR"n"\n"$14"\n";n++;}' \
         TAR_data_NCBI31.txt > yaleBertoneTARSeqs.fa
     ssh pk
     mkdir -p /san/sanvol1/scratch/hg17/TARs/
     cp /cluster/data/hg17/bed/yaleBertoneTars/yaleBertoneTARSeqs.fa \
        /san/sanvol1/scratch/hg17/TARs/
     # Set up to Blat the TAR sequences against hg17
     cd /cluster/data/hg17/bed/yaleBertoneTars
     ls -1 /san/sanvol1/scratch/hg17/TARs/yaleBertoneTARSeqs.fa > tars.lst
     ls -1 /san/sanvol1/scratch/hg17/nib/*.nib > genome.lst
     # output dir
     mkdir psl

     cat << '_EOF_' > template.sub
#LOOP
/cluster/bin/x86_64/blat -repeats=lower -minIdentity=90 -ooc=/san/sanvol1/scratch/hg17/11.ooc $(path1) $(path2) {check out line+ psl/$(root1)_$(root2).psl}
#ENDLOOP
'_EOF_'
    # << for emacs
     gensub2 genome.lst tars.lst template.sub para.spec
     para create para.spec
     para try, para check, para push ...
     para time
# Completed: 46 of 46 jobs
# CPU time in finished jobs:        429s       7.16m     0.12h    0.00d  0.000 y
# IO & Wait Time:                   153s       2.54m     0.04h    0.00d  0.000 y
# Average job time:                  13s       0.21m     0.00h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:              38s       0.63m     0.01h    0.00d
# Submission to last job:           107s       1.78m     0.03h    0.00d
     # sort and then filter
     pslSort dirs raw.psl tmp psl
     # use these parameters as for Genbank alignments of native mRNAs
     # for finished assemblies.
     pslCDnaFilter -minId=0.96 -minCover=0.25 -localNearBest=0.001 \
       -minQSize=20 -minNonRepSize=16 -ignoreNs -bestOverlap \
       raw.psl yaleBertoneTars.psl
#                         seqs    aligns
#             total:     17512   37530
# drop minNonRepSize:     121     254
#     drop minIdent:     3827    14532
#     drop minCover:     571     897
#       weird over:     232     837
#        kept weird:     197     201
#    drop localBest:     2359    3896
#              kept:     17498   17951
     
     # 99.9% were kept. 
     # check how many aligned
     grep '>' yaleBertoneTARSeqs.fa | wc -l
     # 17517
     # 99.89% of the original set of sequences are in this filtered PSL file.
     pslCheck yaleBertoneTars.psl
     # psl is ok

     # load into database
     ssh hgwdev
     cd /cluster/data/hg17/bed/yaleBertoneTars
     hgLoadPsl hg17 yaleBertoneTars.psl

     # Add sequences to /gbdb/hg18 and to seq and extFile tables.
     mkdir -p /gbdb/hg17/yaleTARs/
     ln -s /cluster/data/hg17/bed/yaleBertoneTars/yaleBertoneTARSeqs.fa \
           /gbdb/hg17/yaleTARs/
     hgLoadSeq hg17 /gbdb/hg17/yaleTARs/yaleBertoneTARSeqs.fa
    
     # trackDb.ra entry is in trackDb/human/trackDb.ra and 
     # a description exist already as this track is also on hg18.

######################################################################
## reload tfbsCons table - it was based on a newer version of tfbs names that
#	are not yet public domain (2006-11-03 - Hiram)
    mkdir /cluster/data/hg17/bed/tfbsCons
    cd /cluster/data/hg17/bed/tfbsCons
    cp -p /cluster/store6/weirauch/TFLOC/hg17/tfbsConsSites.bed .
    hgLoadBed -strict hg17 tfbsConsSites \
	-sqlTable=$HOME/kent/src/hg/lib/tfbsConsSites.sql \
	tfbsConsSites.bed -tab
# this leads to a bunch of extra names in Factors
    hgsql -N -e "select name from tfbsConsSites;" hg17 | sort -u > names.new
    hgsql -N -e "select name from tfbsConsFactors;" hg17 \
	| sort -u > names.factors
    comm -13 names.new names.factors > names.extra.factors
for N in `cat extra.names.factors`
do
        echo "delete from tfbsConsFactors where name=\"${N}\";" hg17
        hgsql -e "delete from tfbsConsFactors where name=\"${N}\";" hg17
done

#########################################################
# BUILD GAD TRACK (Re-Re-Done, 12/12/06, Fan)

   mkdir /cluster/store12/gad061211
   rm /cluster/data/gad
   ln -s /cluster/store12/gad061211 /cluster/data/gad

# Receive "GAD-Hg17DATA.txt" from GAD/NIA 
# contact person: Shenoy, Narmada, shenoyn@grc.nia.nih.gov  
   
   hgsql hg17 -e 'drop table gadAll'
   hgsql hg17 <~/src/hg/lib/gadAll.sql
   hgsql hg17 -e 'load data local infile "GAD-Hg17DATA.txt" into table gadAll ignore 1 lines'
   hgsql hg17 -e 'create index geneSymbol on gadAll(geneSymbol(10))'

# create gad table

   hgsql hg17 -N -e \
   'select "chr",chromosome, chromStart, chromEnd, geneSymbol from gadAll
   where chromStart <>0 and chromosome<>""'|\
   sed -e 's/chr\t/chr/' |grep -v "chr\." |grep -v " "|sort -u >gad.bed

   hgLoadBed hg17 gad gad.bed

##########################################################################
#  xxBlastTab - Help filter out unwanted paralogs  (Galt 2007-01-11)
#
# Background: The xxBlastTab tables are made with a simple blastall 
# (blastp with -b 1) which chooses the best match.  Unfortunately this
# means that if there is no proper match it will still pick something
# even though it's probably not orthologous. This is especially a problem
# in organisms like rat knownGene which has only 30% gene coverage. 
# The strategy here is to filter our xxBlastTab using synteny mappings from
# the chains. This is done by simply taking $db.kg and using /gbdb/$db chains
# and pslMap to lift the genes to the target xx assembly.  Then hgMapToGene
# will find which of those mapped ids have good overlap with xx.knownGene.
# The final mapping is then created by doing an inner join between 
# the traditional xxBlastTab and the mapping table produced above.
# Then simply drop the old table and rename the new table.
#
# We are starting with xxBlastTab tables already built in the usual way with
# blastall/blastp, probably with doHgNearBlastp.pl script.
#
# I created a new utility script called synBlastp.csh since I have to do this
# several times. 
#
# we want to update hg17 for rat and mouse, 
# so check ./hgGeneData/Human/hg17/otherOrgs.ra for current settings

ssh hgwdev

synBlastp.csh hg17 rn3
#hg17.rnBlastTab results:
#new number of unique query values:
#10728
#new number of unique target values
#5177
#old number of unique query values:
#24030
#old number of unique target values
#5535

synBlastp.csh hg17 mm7
#new number of unique query values:
#25506
#new number of unique target values
#13462
#old number of unique query values:
#32951
#old number of unique target values
#14803



#####################################################################
##########################################################################
# GenBank gbMiscDiff table (markd 2007-01-10)
# Supports `NCBI Clone Validation' section of mgcGenes details page

   # genbank release 157.0 now contains misc_diff fields for MGC clones
   # reloading mRNAs results in gbMiscDiff table being created.
   ./bin/gbDbLoadStep -reload -srcDb=genbank -type=mrna hg17

############
# UPDATE hg17 knownToVisiGene (DONE galt 2007-02-15)
# Create table that maps between known genes and visiGene database 
    # mapping to other species such as mouse, zebrafish, frog
    # requires visiGene probe track vgImageProbes be created first
    knownToVisiGene hg17 -fromProbePsl=vgImageProbes

#########################################################
# Chimp Paralogy data from Eichlers lab (DONE Heather Feb. 2007)
cd /cluster/data/hg17/bed/eichler
hgLoadBed hg17 chimpParalogy chimpParalogy.bed -tab -sqlTable=chimpParalogy.sql

############################################################################
# Reload CCDS (ccdsInfo, ccdsGene, ccdsKgMap) (2007-03-02 markd)

    cd /cluster/data/genbank/data/ccds/
    ftp ftp-private.ncbi.nih.gov (user ccds, needs password)
    get CCDS.20070228.tar.gz
    mkdir /scratch/tmp/ccds
    cd /scratch/tmp/ccds
    tar -zxf /cluster/data/genbank/data/ccds/CCDS.20070228.tar.gz

    # import ccds database tables
    /cluster/data/genbank/bin/x86_64/ccdsImport ccds data/*.txt

    # create and load ccdsGene and ccdsInfo tables from imported database
    /cluster/data/genbank/bin/x86_64/ccdsMkTables -loadDb ccds hg17 ccdsInfo ccdsGene
    /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap  -db=hg17 -loadDb ccdsGene knownGene ccdsKgMap
    checkTableCoords hg17 -verbose=2 ccdsGene
    joinerCheck -database=hg17 -identifier=ccdsGeneId ~/compbio/genbank/kent/src/hg/makeDb/schema/all.joiner

    # build initial version of ccdsMgcMap table, updated by nightly  genbank update
    /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap -loadDb -db=hg17 ccdsGene mgcGenes ccdsMgcMap
    
    # load trackDb
    cd kent/src/hg/makeDb/trackDb
    make alpha
    # check in browser

    # request push of 
        ccdsGene
        ccdsInfo
        ccdsKgMap
        ccdsMgcMap
    # << emacs

#####################################################
# Vista Enhancers (galt 2007-02-23 done)
#
# Vista from Lawrence-Berkeley has assayed
# 301 human conserved non-coding intra- and inter-
# genic elements for their ability to promote
# lacZ in mouse embryos.  A positive looks like
# a mouse with a purple spine.
#
# They provided a custom track with two tracks for pos and neg.
# http://enhancer-test.lbl.gov/cgi-bin/customTrack.cgi
# I am combining the tracks into one with high score for pos.
#
cd /cluster/data/hg17/bed
mkdir vistaEnhancers
cd vistaEnhancers
wget -O custTrk "http://enhancer-test.lbl.gov/cgi-bin/customTrack.cgi"
cat custTrk | head -116 | tail +2 > pos
cat custTrk | tail +118 > neg
cat pos | gawk '{print $1"\t"$2"\t"$3"\t"$4"\t900"}' > bed5
cat neg | gawk '{print $1"\t"$2"\t"$3"\t"$4"\t200"}' >> bed5
wc -l bed5
#301 bed5
hgLoadBed hg17 vistaEnhancers bed5
#Loaded 301 elements of size 5

# add to human/trackDb.ra
track vistaEnhancers
shortLabel Vista Enhancers
longLabel Vista HMR-Conserved Non-coding Human Enhancers from LBNL
group regulation
priority 93
visibility hide
color 50,70,120
type bed 5 .
useScore 1
url http://enhancer-test.lbl.gov/cgi-bin/imagedb.pl?form=presentation&show=1&experiment_id=$$

###
# UPDATES (2007-10-18, conodera)
# see also /projects/compbiousr/wet/browser/vista_enhancer/17Oct2007/Makefile

cd /projects/compbiousr/wet/browser/vista_enhancer/

# download data file from the vista browser (coordinates are for hg17)
# http://enhancer.lbl.gov/cgi-bin/imagedb.pl?show=1;search.result=yes;form=search;search.form=no;action=search;search.sequence=1
# save as enhancerbrowser.datadownload.txt

# give elements with positive label a score of 900, 
# give elements with negative label a score of 200.
# print to 5-field bed file
vista_enhancer.hg17.txt: enhancerbrowser.datadownload.txt
        grep ">" $< \
        | sed -e 's/>//' \
        | tr :- '       ' \
        | sed -e 's/positive/900/'\
        | sed -e 's/negative/200/' \
        | awk '{print $$1"\t"$$2"\t"$$3"\telement_"$$6"\t"$$8}' \
        > $@; \
hgLoadBed hg17 vistaEnhancers vista_enhancer.hg17.txt;
# loaded 446 elements of length 5


#########################################################################
# EPONINE-TSS (TRANSCRIPTON START SITE) PREDICTION 
# (DONE, 2007-03-08, hartera)
# The Eponine software is version 2 and has not changed in several years
# (contact: Thomas Down at Sanger, td2@sanger.ac.uk). The version downloaded
# for hg16 should be the same as the current version but download again just
# to check. The application includes the TSS model file: eponine-tss2.xml 

     ssh kkstore02
     # Eponine runs fine on 2.5Mb contig, but barfs on much larger contig;
     # chop up sequence at gaps into ~2.5Mb chunks for cluster run.
     mkdir /san/sanvol1/scratch/hg17/chunks
     cd /cluster/data/hg17
     foreach f (?{,?}/NT_*/NT_??????.fa)
       set ctg = $f:t:r
       /cluster/bin/x86_64/faSplit -minGapSize=10 \
        -lift=/san/sanvol1/scratch/hg17/chunks/${ctg}.lft \
        gap $f 2500000 /san/sanvol1/scratch/hg17/chunks/${ctg}.chunk
     end
     # seems to ignore the chunk part of the file name
     mkdir /cluster/data/hg17/bed/eponine
     cd /cluster/data/hg17/bed/eponine
     wget --timestamping \
       http://www.sanger.ac.uk/Software/analysis/eponine/eponine-scan.jar
     # file has the same date and same size as the one downloaded for hg16
     # the script requires all of the path setting found in my .tcshrc file.
     # Using only set path = (/usr/java/jre1.5.0_06/bin $path)
     # as in the doEpo file for hg16 does not work.
     cat << '_EOF_' > doEpo
#!/bin/csh -ef
set path = (/usr/java/jre1.5.0_06/bin /bin /usr/bin /usr/X11R6/bin \
             /usr/local/bin . /cluster/home/hartera/bin/x86_64 \
             /cluster/bin/x86_64 /projects/compbio/bin/x86_64 \
             /projects/compbio/bin /projects/compbio/bin/x86_64-linux \
             /cluster/bin/scripts)
java -jar ./eponine-scan.jar -threshold 0.999 -seq $1 > $2
'_EOF_'
     # << emacs
     chmod a+x doEpo
     cp /dev/null jobList
     foreach f (/san/sanvol1/scratch/hg17/chunks/NT*.fa)
        echo "./doEpo {check in line+ $f} {check out exists out/$f:t:r.gff}" \
      >> jobList
     end
     mkdir out
     ssh pk
     cd /cluster/data/hg17/bed/eponine
     /parasol/bin/para create jobList
     /parasol/bin/para try, check, push, check etc.....
     /parasol/bin/para time
# Completed: 1415 of 1415 jobs
# CPU time in finished jobs:     104501s    1741.68m    29.03h    1.21d  0.003 y
# IO & Wait Time:                  6594s     109.91m     1.83h    0.08d  0.000 y
# Average job time:                  79s       1.31m     0.02h    0.00d
# Longest running job:                0s       0.00m     0.00h    0.00d
# Longest finished job:             127s       2.12m     0.04h    0.00d
# Submission to last job:           488s       8.13m     0.14h    0.01d
     # lift chunks -> contigs
    mkdir contigs/
    foreach l (/san/sanvol1/scratch/hg17/chunks/*.lft)
      set ctg = $l:t:r
      liftUp contigs/$ctg.gff $l warn out/${ctg}*.gff
    end
    # lift contigs -> chrom
    liftUp eponine.gff /cluster/data/hg17/jkStuff/liftAll.lft \
           warn contigs/NT_*.gff
    # Translate to bed 4 + float-score -- it would be a shame to lose 
    # those scores in genePred or bed 5 (int score)
    awk 'BEGIN {i=0;} \
         {printf "%s\t%d\t%d\t%s.%d\t%s\t%s\n", $1, $4-1, $5, $1, i, $6, $7; \
          i = i + 1;}' \
      eponine.gff > eponine.bed
    # load up
    ssh hgwdev
    cd /cluster/data/hg17/bed/eponine
    sed -e 's/bed6FloatScore/eponine/g' \
      $HOME/kent/src/hg/lib/bed6FloatScore.sql > eponine.sql
    hgLoadBed hg17 eponine eponine.bed -tab -sqlTable=eponine.sql
    # Loaded 61013 elements of size 6
    # trackDb.ra entry and eponine.html already exist in trackDb directory.

###########################################################################
# ACEScan Track (DONE 2007-03-15  Andy

ssh hgwdev
cd /cluster/data/hg17/bed
mkdir acescan
cd acescan/
cp ~/acescan.gff .
tail +2 acescan.gff > acescan.nh.gff
ldHgGene -out=gp hg17 acescan acescan.nh.gff 
rm *.gff
ldHgGene -predTab hg17 acescan acescan.hg17.gp

###########################################################################
# augustusHints track (DONE 2007-4-5 Mario)

mkdir -p /cluster/data/hg17/bed/augustus/usingHints/predictions/Trefseq.hmRNA.hsEST.R.Xp.final
cd /cluster/data/hg17/bed/augustus/usingHints/predictions/Trefseq.hmRNA.hsEST.R.Xp.final
wget http://augustus.gobics.de/predictions/hg17/usingEvidence/augustus.hg17.Trefseq.hmRNA.hsEST.R.Xp.gff
wget http://augustus.gobics.de/predictions/hg17/usingEvidence/augustus.hg17.Trefseq.hmRNA.hsEST.R.pep.aa
ldHgGene -bin hg17 augustusHints augustus.hg17.Trefseq.hmRNA.hsEST.R.Xp.gff
hgPepPred hg17 generic augustusHintsPep augustus.hg17.Trefseq.hmRNA.hsEST.R.Xp.pep.aa

###########################################################################
# augustus de novo track (DONE 2007-4-5 Mario)

mkdir -p /cluster/data/hg17/bed/augustus/usingHints/predictions/Xp.RA.it
cd /cluster/data/hg17/bed/augustus/usingHints/predictions/Xp.RA.it
wget http://augustus.gobics.de/predictions/hg17/deNovo/augustus.hg17.Xp.RA.it.gff
wget http://augustus.gobics.de/predictions/hg17/deNovo/augustus.hg17.Xp.RA.it.pep.aa
ldHgGene -bin hg17 augustusXRA augustus.hg17.Xp.RA.it.gff
hgPepPred hg17 generic augustusXRAPep augustus.hg17.Xp.RA.it.pep.aa


###########################################################################
# SwitchDB TSS Track (DONE 2007-04-12 Andy)

ssh hgwdev
mkdir /cluster/data/hg17/bed/switchDbTss
cd /cluster/data/hg17/bed/switchDbTss
# (obtained from Nathan Trinklein <nathant@switchgeargenomics.com>)
cp ~/all_tss_switchdb_psgene.gz .
gunzip all_tss_switchdb_psgene.gz
cat << "EOF" > reformat.awk
BEGIN{FS="\t"}
{
if (NR > 1)
   {
   if ($9 !~ "^PSEUDO.*")
       {
       pseudo = "none";
       }
   else
       {
       pseudo = $9;
       }
   printf("%s\t%d\t%d\t%s\t1000\t%s\t%s\t%s\t%s\t%s\t%s\n", $2, $8, $8+1, $6, $5, $7, $1, $3, $4, pseudo); 
   }
}
EOF
awk -f reformat.awk all_tss_switchdb_psgene > switchDbTss.bed
ln -s ~/kent/src/hg/lib/switchDbTss.sql
hgLoadBed -sqlTable=switchDbTss.sql hg17 switchDbTss switchDbTss.bed

############################################################################
# enable ORFeome track build.  (markd 2007-05-02)
    cd ~/kent/src/hg/makeDb/genbank
    cvs update -d etc
    # edit etc/genbank.conf to add 
        hg17.orfeomeTables.hgwdev = yes
        hg17.orfeomeTables.hgwbeta = yes
    # will need to enable for rr later.  In the future, this can just be enabled
    # as part the normal genbank build.  Change above to:
        hg18.orfeomeTables.default = yes
    
###########################################################################
# Transcriptome Phase 3 tracks (Andy 2007-06-10)

ssh hgwdev
bash
cd /san/sanVol1/scratch/andy
mkdir transcriptome
cd transcriptome/
cp /var/ftp/encode/Affy_transcriptome_phase3.tar .
tar xfv Affy_transcriptome_phase3.tar 
find . -name '*.bz2' -exec bunzip2 '{}' \;
cat > processWig.sh << "EOF"
#!/bin/bash

theDir=`dirname $1`;
theFile=`basename $1`;
table=affyTxnPhase3${theFile%.sig.wig};
tmp=/scratch/tmp/trans3rdPhase.$$

mkdir $tmp
cp $1 $tmp
pushd $tmp
head -n1 $theFile > $table.sig.track.txt
tail +2 $theFile > tmp; mv tmp $theFile
wigEncode $theFile $table.wig $table.wib
popd
cp $tmp/${table}.* $theDir
rm -rf $tmp
EOF
chmod +x processWig.sh
cat > gsub << "EOF"
#LOOP
./processWig.sh {check in line+ $(path1)} {check out exists $(dir1)/$(root1).track.txt}
#ENDLOOP
EOF
find . -name '*.sig.wig' > wig.lst
gensub2 wig.lst single gsub spec
ssh pk
cd /san/sanVol1/scratch/andy/transcriptome
para create spec
para push
exit
cd /cluster/data/hg17/bed
mkdir transcriptome3rdPhase/{wig,wib,bed}
cd transcriptome3rdPhase/wib/
cp /san/sanVol1/scratch/andy/transcriptome/graphs/human_{long,short}_rna/affyTxnPhase3*.wib .
pushd /gbdb/hg17/wib
ln -s /cluster/data/hg17/bed/transcriptome3rdPhase/wib/* .
popd
cd ../wig/
cp /san/sanVol1/scratch/andy/transcriptome/graphs/human_{long,short}_rna/affyTxnPhase3*.wig .
for f in *; do
   hgLoadWiggle hg17 ${f%.wig} $f
done
cd ../bed
for f in /san/sanVol1/scratch/andy/transcriptome/transfrags/human_{long,short}_rna/*; do
   newName=`basename $f`;
   newName=${newName%.bz2};
   bzcat $f | tail +2 > $newName;
   tab=affyTxnPhase3Frags${newName%.bed};
   hgLoadBed hg17 $tab $newName;
done

#######################################################################
# CLEANUP OF DANRER1 BLASTZ SWAP (DONE, 2007-06-25, hartera)
   ssh kkstore02
   cd /cluster/data/hg17/bed/blastz.danRer1.swap/axtChain
   rm -r run1
   cd ../mafNet.new
   gzip *.maf
   # we don't tend to keep the Blastz PSLs anymore and this is an old 
   # zebrafish assembly so remove these.
   cd ../
   rm -r pslChrom
   # this removed 1.2 G of data.

#######################################################################
# CLEANUP OF ACEMBLY_050217 DATA (DONE, 2007-06-25, hartera)
   ssh kkstore02
   cd /cluster/store5/gs.18/build35/bed/acembly_050217
   rm GeneCheck.out GeneCheck2 acembly acembly.chk acembly.details \
      chrom.out genePred.tab hg16.name hg16Pep.name
   cd acembly_gene_lists
   rm test transcripts.names *.bak main_gene.list.IDsort mp.ids mp.sort ptest \
      maintest gid.tab gid.tab.sort genesGffs.ids genesGffs.ids.uniq 
   cd ../
   # remove fasta files as included in gzipped tar file
   rm -r acembly.ncbi_35.genes.proteins.fasta 
   cd acembly.ncbi_35.genes.gff
   gzip *.gff 

#######################################################################
# CLEANUP OF DANRER2 BLASTZ SWAP (DONE, 2007-06-25, hartera)
   ssh kkstore02
   cd /cluster/store5/gs.18/build35/bed/blastz.danRer2.2004-12-08
   # remove old axtChrom directory
   rm -r axtChrom.orig
   cd axtChain
   # chain directories can be recreated from all.chain files so remove
   rm -r chain chainAR
   # gzip net files
   gzip net/*.net
   # gzip .over files
   gzip over/*.over
   # removed ~1.3 G data 

#############################################################################
# Duke DNaseI HS (2007-06-26 kate)
#
#  Submitted by Terry Furey <terry.furey@duke.edu>
#  in collaboration with Greg Crawford
# Resubmitted 9/26/07 from FTP site
# Resubmitted 10/25/07 from FTP site

    ssh kkstore02
    cd /cluster/data/hg17/bed

    # download 19GB archive from Duke site, password protected,
    # user=ucsc, password=dnase
    mkdir -p dukeDnase/2007-10-25/lab
    cd dukeDnase/2007-10-25/lab

    sftp ucsc@sftp.igsp.duke.edu
    mget *
    # dukeDnaseHsCd4.bed
    # dukeDnaseHsCd4Wiggle.tgz

    # unpack and load wiggle (signal) data
    nice tar xvfz dukeDnaseHsCd4Wiggle.tgz
    # packaged as chr*_dukeDnaseHsCd4Wiggle.out
    # fixedStep 1 files

    # create wiggle and load into database
    cd ..
    cat lab/chr*.out | nice wigEncode stdin \
        dukeDnaseCd4Signal.wig dukeDnaseCd4Signal.wib >&! wigencode.log &
        # upper limit 25.74, lower limit -0.66

    ssh hgwdev
    cd /cluster/data/hg17/bed/dukeDnase/2007-10-25
    rm -f /gbdb/hg17/wib/dukeDnaseCd4Signal.wib
    ln -s /cluster/data/hg17/bed/dukeDnase/2007-10-25/dukeDnaseCd4Signal.wib \
            /gbdb/hg17/wib
    nice hgLoadWiggle hg17 dukeDnaseCd4Signal -pathPrefix=/gbdb/hg17/wib \
                dukeDnaseCd4Signal.wig

    # load bed file (sites)
    ssh hgwdev
    cd /cluster/data/hg17/bed/dukeDnase/2007-10-25/
    set table = dukeDnaseCd4Sites
    sed "s/bed5FloatScore/$table/" ~/kent/src/hg/lib/bed5FloatScore.sql > \
            $table.sql
    hgsql hg17 -e "DROP TABLE IF EXISTS $table"
    hgsql hg17 < $table.sql
    hgLoadBed -sqlTable=$table.sql hg17 $table lab/dukeDnaseHsCd4.bed
        # Loaded 95723 elements of size 6
        # min value:  0.000103164
        # max value: 25.7442
    #textHistogram -col=5 lab/dukeDnaseHsCd4.bed -binSize=50 
    300 ******************************** 11789
    350 ************************************************************ 22253
    400 ********************************************* 16854
    450 ********************************* 12333
    500 ************************* 9179
    550 ********************* 7870
    600 ************* 4987
    650 ********* 3271
    700 ******** 2789
    750 ****** 2153
    800 **** 1303
    850 ** 567
    900 * 219
    950  85
    1000  71


###########################################################################
# Stanford ChIP-seq (Apr - July 2007, Heather)
# Submitted 2007-03-14 by David Johnson <seasquirtdoctor@gmail.com>
# 25bp tags (Solexa sequencing of IP fragments)
# genome-wide, but funded by ENCODE, hence the location of the data

    ssh hgwdev
    cd /cluster/data/encode/stanford
    mkdir -p 2007-03-14/lab
    cd 2007-03-14/lab
    sort NRSF_chipseq_hg17.bed > data.bed
    sort NRSF_chipseq_control_hg17.bed > control.bed
    fix.pl < data.bed > fix.bed
    fix.pl < control.bed > control_fix.bed
    hgLoadBed hg17 stanfordNRSFEnriched fix.bed -tab
    hgLoadBed hg17 stanfordNRSFControl control_fix.bed -tab

############################################################################
# Stanford ChIP/chip
# Submitted 2007-07-11 by David Johnson (seasquirtdoctor@gmail.com)
# Replaces submission from 2007-03-23
# 12 subtracks
# genome-wide, but funded by ENCODE, hence the location of the data

    ssh hgwdev
    cd /cluster/data/encode/stanford/2007-07-11/lab
    # Dave gave us bed 5, we need bed 4
    ./shrink.sh
    ./load.sh

#########################################################################
# REGULATORY POTENTIAL 7X UPDATED (DONE - 2007-08-01 - Hiram)
    #	download data from "James Taylor" <james@bx.psu.edu>
    ssh kkstore02
    mkdir /cluster/data/hg17/bed/regPotential7X.update
    cd /cluster/data/hg17/bed/regPotential7X.update
    ## In theory, only chr4, chr8, chr9 and chrY have updated, fetch them
    ## all and verify with ../regPotential7X
for C in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y
do
wget --timestamping \
"http://www.bx.psu.edu/~james/esperr_rp_7way_scores/genome_scores_hg17/chr${C}.scores.truncated.bz2"
echo "DONE - chr${C}.scores.bz2"
done

    #	create download gzip files from the bz2 files:
    time for F in chr*.scores.truncated.bz2
    do
	C=`echo $F | awk -F'.' '{print $1}'`
	echo -n "${C}.regPotential7X.hg17.gz working ... "
	bzcat ${F} | gzip > ${C}.regPotential7X.hg17.gz
	echo "done"
    done

    time for C in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 X Y
    do
	zcat chr${C}.regPotential7X.hg17.gz
    done | wigEncode -noOverlap  stdin regPotential7X.wig regPotential7X.wib
    #	Converted stdin, upper limit 1.00, lower limit -0.00
    #	real    16m51.215s

    #	Loading the table on hgwdev
    ssh hgwdev
    cd /cluster/data/hg17/bed/regPotential7X.update
    mkdir /gbdb/hg17/wib/061116
    ln -s /cluster/data/hg17/bed/regPotential7X.update/regPotential7X.wib \
	/gbdb/hg17/wib/061116/regPotential7X.wib
    #	using the tmpDir is faster since it is on local disk and it will
    #	clean up any temporary .tab file it creates there
    time nice -n +19 hgLoadWiggle -tmpDir=/scratch/tmp \
	-pathPrefix=/gbdb/hg17/wib/061116 hg17 regPotential7X regPotential7X.wig
    #	real    0m40.523s

    #	How about a histogram of the data.
    ssh kolossus
    cd /cluster/data/hg17/bed/regPotential7X.update
    time nice -n +19 hgWiggle -verbose=2 -doHistogram -hBinSize=0.01 \
      -hBinCount=100 -hMinVal=0.0 -db=hg17 regPotential7X > histogram.data 2>&1
    #	real    3m3.829s
    #	73 % of the data values are zero


    # renaming file directory -- kuhn 08-17-2007
    cd /gbdb/hg17/wib
    mv 061116 regPot061116
    hgsql -e " update regPotential7X SET file = \
      /gbdb/hg17/wib/regPot061116/regPotential7X.wib" hg17
    Query OK, 2366123 rows affected (31.46 sec)
    Rows matched: 2366123  Changed: 2366123  Warnings: 0

###########################################################################
## Create gc5Base download raw data file (DONE - 2007-08-29 - Hiram)
    ssh kkstore02
    cd /cluster/data/hg17/bed/gc5Base
    hgGcPercent -wigOut -doGaps -file=stdout -win=5 \
	hg17 /cluster/data/hg17/hg17.2bit 2> /dev/null \
	| gzip > hg17.gc5Base.txt.gz
    ssh hgwdev
    mkdir /usr/local/apache/htdocs/goldenPath/hg17/gc5Base
    cd /usr/local/apache/htdocs/goldenPath/hg17/gc5Base
    ln -s /cluster/data/hg17/bed/gc5Base/hg17.gc5Base.txt.gz .

############################################################################
# INDEL-BASED CONSERVATION TRACK (DONE, 2007-10-02 - 2007-10-03, hartera)
# Data from the Gerton Lunter (gerton.lunter@anat.ox.ac.uk), MRC 
# Functional Genetics Unit, University of Oxford, United Kingdom.
# Data is from the paper:
# Lunter G, Ponting CP and Hein J Genome-wide identification of human
# functional DNA using a neutral indel model. PLoS Comput Biol. 2006
# Jan;2(1):e5.
    ssh kkstore02 
    mkdir -p /cluster/data/hg17/bed/consIndels/data
    cd /cluster/data/hg17/bed/consIndels/
    # Add a README.indels with the e-mail from Gerton Lunter, copy over 
    # from hg18 condIndels
    cp /cluster/data/hg18/bed/consIndels/README.indels .
    # get the data
    cd data
    wget --timestamping \
         http://wwwfgu.anat.ox.ac.uk/~gerton/IPS/IPSs.zip
    # 15 Mb zip file in GFF format. This contains data for hg17
    # comparing it to mm5 (NCBI Build 33) and 
    # canFam1 (Broad Institute, July 2004). The chr*.mm5.GFF data is old
    # data that can be removed.
    unzip IPSs.zip    
    cd /cluster/data/hg17/bed/consIndels
    rm ./data/*mm5.GFF
    foreach f (./data/*.GFF)
       set r = $f:r
       echo $r
       grep -v "track" $f > ${r}NoHeader.gff
    end
  
    # strip off the end of the name e.g. IGS0001:p=.26
    # so that the name displayed is short - IGS0001.1. The score field
    # is used to determine colouring and this is calculated from FDR 
    ssh kkstore02 
    cd /cluster/data/hg18/bed/consIndels
    perl -pi.bak -e \
         's/(IGS[0-9a-z]+\.?[0-9XY]*):p=?<?\.[0-9]+/$1/'  \
         ./data/chr*NoHeader.gff
    # check this looks ok then clean up
    rm *.bak
    # makes sense to store this as a BED5 table in order to use the score
    # for display.
    foreach f (./data/*NoHeader.gff)
       awk 'BEGIN {FS="\t"} {OFS="\t"} {print $1,$4,$5,$9,$6}' $f \
       >> consIndelsHg17Mm5CanFam1.bed
    end
 
    # load data
    ssh hgwdev
    cd /cluster/data/hg17/bed/consIndels
    hgLoadBed hg17 consIndelsHg17Mm5CanFam1 consIndelsHg17Mm5CanFam1.bed
    # Loaded 593298 elements of size 5
    
    # Get the IDs, posterior probabilities (p) for the segment being neutral, 
    # and the FDR from the original GFFs for a separate table. Some items 
    # have p<.001. Can not do Table Browser queries restricting 
    # p to <, =, or > a specified value unless all values are floats.
    # Contacted the data contributor, Gerton Lunter, and he said it would be 
    # ok to change all p<.001 to p=0.0005
    ssh kkstore02
    
    cd /cluster/data/hg17/bed/consIndels/
    awk '{if ($1 !~ /random/) print $1;}' /cluster/data/hg17/chrom.sizes \
        | sed -e 's/chr//' | sort -n > chrom.lst
    grep -v 'hap' chrom.lst > tmp2
    tail +4 tmp2 > tmp3
    echo "X\nY\n" >> chrom.lst
    rm tmp2 tmp3 
    # chrom.lst has a list of chroms 1-22, then X and Y
    foreach c (`cat chrom.lst`)
       echo $c
       foreach f (./data/chr${c}.GFF)
          echo $f
          awk 'BEGIN {FS="\t"} {OFS="\t"}{if ($9 ~ /IGS/) print $9,$6;}' $f \
              | sed -e 's/:/\t/' \
              | sed -e 's/p=\./0\./' | sed -e 's/p<\.001/0\.0005/' \
              >> consIndelsConf.txt
       end
    end
    # Add the FDR. 
    # For this set, there is no false discovery rate (FDR) field but it 
    # can be related to the score. If score is 999 then FDR is 1% (0.01) and
    # if score is 500 then FDR is 10% (0.10). Score is in column 6.
    # there are no GFF files for the haplotype chroms
   
    awk 'BEGIN {FS="\t"} {OFS="\t"} {if ($3 ~ /500/) print $1, $2, "0.10"; else if ($3 ~ /999/) print $1, $2, "0.01";}' consIndelsConf.txt > consIndelsHg17Mm5CanFam1Conf.txt 

    # Create a table definition for the table of identifier,  posterior 
    # probability and false discovery rate (FDR). Already created for hg18
    # track (see hg18.txt). It is $HOME/kent/src/hg/lib/itemConf.as.
   
    ssh hgwdev
    cd /cluster/data/hg17/bed/consIndels
    hgLoadSqlTab hg17 consIndelsHg17Mm5CanFam1Conf \
         $HOME/kent/src/hg/lib/itemConf.sql \
         consIndelsHg17Mm5CanFam1Conf.txt
    # check that all itesm are in this table.
    hgsql -N -e 'select distinct(name) from consIndelsHg17Mm5CanFam1;' hg17 \
         | sort > consIndels.names.sort
    hgsql -N -e 'select distinct(id) from consIndelsHg17Mm5CanFam1Conf;' hg17 \
         | sort > consIndels.idsfromConf.sort
    wc -l *.sort
    # 593298 consIndels.idsfromConf.sort
    # 593298 consIndels.names.sort

    comm -12 consIndels.names.sort consIndels.idsfromConf.sort | wc -l
    # 593298
    # so all element IDs are in both tables.
    # cleanup
    rm ./data/*.bak *.sort
    
    # add trackDb/human/hg17/trackDb.ra entry and add description that
    # was written by the data contributor. Add code to hgc.c to display
    # the posterior probability and the FDR on the details page for
    # track elements. Gerton Lunter provided a description for the data
    # on 2007-09-12.
    cd ~/kent/src/hg/makeDb/trackDb/human/hg17
    cp ../hg18/consIndelsHg18Mm8CanFam2.html consIndelsHg17Mm5CanFam1.html
    # check this is correct and add trackDb.ra track entry and search.  

##############################################################
# NIMH Bipolar Genome Graphs built-in (DONE 2007-10-04 Galt)
#
#  See hg18.txt for details.
#############################################################

#############################################################
# CCC Genome Graphs (DONE 2007-Sept Andy)
# 
# See hg18 make doc.

###############################################################
# Affy Transcriptome Phase 3 chrY fix (DONE 2007-12-10, Andy)

ssh kkstore05
cd /cluster/store12/hg17/bed/affyTxnPhase3/raw
zcat sRNA.affyTxnPhase3HeLaBottomStrand.wig.gz | grep -n chrY
#256994657:variableStep chrom=chrY span=1
zcat sRNA.affyTxnPhase3HeLaBottomStrand.wig.gz | head -n256994656 | gzip -c >tmp.wig.gz
mv tmp.wig.gz sRNA.affyTxnPhase3HeLaBottomStrand.wig.gz
zcat sRNA.affyTxnPhase3HeLaTopStrand.wig.gz | grep -n chrY 
256994657:variableStep chrom=chrY span=1
zcat sRNA.affyTxnPhase3HeLaTopStrand.wig.gz | head -n256994656 | gzip -c > tmp.wig.gz
mv tmp.wig.gz sRNA.affyTxnPhase3HeLaTopStrand.wig.gz
ssh kolossus
cd /cluster/store12/hg17/bed/affyTxnPhase3/raw
wigEncode sRNA.affyTxnPhase3HeLaBottomStrand.wig.gz affyTxnPhase3HeLaBottomStrand.{wig,wib}
wigEncode sRNA.affyTxnPhase3HeLaTopStrand.wig.gz affyTxnPhase3HeLaTopStrand.{wig,wib}
mv *.wig /cluster/data/hg17/bed/affyTxnPhase3/wig/
mv *.wib /cluster/data/hg17/bed/affyTxnPhase3/wib/
ssh hgwdev
cd /cluster/data/hg17/bed/affyTxnPhase3/wig
hgLoadWiggle hg17 affyTxnPhase3HeLaTopStrand{,.wig}
hgLoadWiggle hg17 affyTxnPhase3HeLaBottomStrand{,.wig}

###########################################################################
# Reload CCDS (2007-12-12 markd)
    # import ccds database as described in ccds.txt
    set db=hg17
    # create and load ccdsGene and ccdsInfo tables from imported database
    /cluster/data/genbank/bin/x86_64/ccdsMkTables -loadDb ccds ${db} ccdsInfo ccdsGene

    # ccdsKgMap
    /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap -db=${db} -loadDb ccdsGene knownGene ccdsKgMap

    # build initial version of ccdsMgcMap table, updated by nightly genbank update
    /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap -db=${db} -loadDb ccdsGene mgcGenes ccdsMgcMap

    checkTableCoords ${db} -verbose=2 ccdsGene
    # update all.jointer to include ${db} in ccdsDb
    joinerCheck -database=${db} -identifier=ccdsGeneId ~/compbio/genbank/kent/src/hg/makeDb/schema/all.joiner
    # request push of 
        ccdsGene
        ccdsInfo
        ccdsKgMap
        ccdsMgcMap
    # << emacs

############################################################################
# ADD LINKS TO GENETESTS ON hgGene DETAILS PAGE (DONE 12/12/07 Fan)

See hg18.txt for details.

############################################################################
# Reload CCDS (2008-02-01 markd)
    # import ccds database as described in ccds.txt
    set db=hg17
    # create and load ccdsGene and ccdsInfo tables from imported database
    /cluster/data/genbank/bin/x86_64/ccdsMkTables -loadDb ccds ${db} ccdsInfo ccdsGene

    # ccdsKgMap
    /cluster/data/genbank/bin/x86_64/mkCcdsGeneMap -db=${db} -loadDb ccdsGene knownGene ccdsKgMap

    checkTableCoords ${db} -verbose=2 ccdsGene
    # update all.jointer to include ${db} in ccdsDb
    joinerCheck -database=${db} -identifier=ccdsGeneId ~/compbio/genbank/kent/src/hg/makeDb/schema/all.joiner
    # request push of 
        ccdsGene
        ccdsInfo
        ccdsKgMap
    # << emacs

############################################################################
# CREATE huge TABLE FOR HuGE LINK (DONE 3/6/08, Fan)

# See the HuGE section in hg18.txt for details.
############################################################################


############################################################################
# DGV V10 (DATABASE OF GENOMIC VARIANTS) (DONE 11/10/10 angie - color change 2/22/11 #2917)
# DGV V9 done 3/26/10
# DGV V8 done 8/12/09 (changed color of inverted 11/05/09 kuhn)
# DGV V7 done 3/11/09
# DGV V6 thin regions dropped 2/23/09
# DGV V6 with useless thin regions done 11/12/08
# DGV V5 done 8/11/08
# DGV V4 done 5/9/08
    ssh hgwdev
    mkdir /hive/data/genomes/hg17/bed/dgv.v10
    cd /hive/data/genomes/hg17/bed/dgv.v10
    wget --timestamping \
      http://projects.tcag.ca/variation/downloads/variation.hg17.v10.nov.2010.txt
    wget --timestamping \
      http://projects.tcag.ca/variation/downloads/indel.hg17.v10.nov.2010.txt
    # shuffle fields into bed9+ w/itemRgb (input has one start coord==0, but min 
    # nonzero size of 99 not 100 implies most coords are 1-based):
    set purple = "200,0,200"
    set red = "200,0,0"
    set blue = "0,0,200"
    set brown = "139,69,19"
    foreach f (*.v10.*.txt)
      tail -n +2 $f \
      | perl -wpe 'chomp; \
        ($id, $landmark, $chr, $start, $end, $varType, \
         undef, undef, undef, $ref, $pmid, $method, \
         $gain, $loss, undef, undef, $sample) = split("\t"); \
        $id =~ s/^Variation_//; \
        $start-- unless ($start == 0); \
        $landmark = "" if ($landmark =~ /^chr.*\d\.\.\d/); \
        $rgb = ($varType =~ /^Inv/) ? "'$purple'" : "'$brown'"; \
        if ($gain ne "" || $loss ne "") { \
          $gain =~ s/^(NA)? ?$/0/;  $loss =~ s/^(NA)? ?$/0/; \
          $rgb = "'$blue'" if ($gain > 0 && $loss == 0); \
          $rgb = "'$red'" if ($loss > 0 && $gain == 0); \
        } \
        $_ = join("\t", $chr, $start, $end, $id, 0, "+", \
                  $start, $start, $rgb, $landmark, $varType, \
                  $ref, $pmid, $method, $sample) . "\n";' \
          > $f:r.bed
    end
    hgLoadBed hg17 dgv *.bed \
      -onServer -sqlTable=$HOME/kent/src/hg/lib/dgv.sql -tab
#Loaded 23423 elements of size 15
    # That's a decrease of 1.  Jeff MacDonald previously explained that
    # hg17 is not getting anything new because all of the new studies 
    # give hg18 or hg19 coords.  They map forward (hg18 -> hg19) but
    # not backwards.

# 11-04-2009 color change from brown to magenta:
# old color
# 6553700	Inversion  (100,0,100)
# new:
# 13107400	Inversion  (200,0,200)

# 2/22/11 color change (Bug #2917): swap blue and red; green -> brown

############################################################################
# KIDD/EICHLER DISCORDANT CLONE ENDS (DONE 6/10/08 angie)
# 8/11/08: Added kiddEichlerToNcbi (ID xref table).
    ssh kkstore02
    mkdir /cluster/data/hg17/bed/kiddEichlerDiscordant
    cd /cluster/data/hg17/bed/kiddEichlerDiscordant
    wget --user=uuuu  --password=ppppppp \
      http://eichlerlab.gs.washington.edu/kiddj/downloads/fosmids.hg17.tgz
    tar xvzf fosmids.hg17.tgz
    cd bd35
    # 8 clone-end linkedFeaturesSeries tracks and one bed custom track.
    # bed has illegal coords (maybe for unplaced ends?).

    # Load the tracks (translate bacEndPairs format to bed12):
    ssh hgwdev
    cd /cluster/data/hg17/bed/kiddEichlerDiscordant/bd35
    foreach f (abc*.txt)
      set track = `echo $f:r \
        | perl -wpe 's/^(G|abc)(\d+)discordant/kiddEichlerDisc\u$1$2/ || die;'`
      if ($status != 0) break
      perl -wpe 'next if s/^#.*\n$//; \
        ($c, $s, $e, $n, $sc, $st, undef, $bs, $bSt, $bSz)=split; \
        @bSts = split(",", $bSt);  @bSzs = split(",", $bSz); \
        $s--; \
        if ($n =~ /transchr/) { \
          $bs = 1; \
          $#bSts = 0;  $#bSzs = 0; \
          $bSts[0]--;  $e--; \
          $bSts[0] -= $s; \
        } elsif ($n =~ /OEA/) { \
          $bSts[0]--; \
          die "bSts[0] $bSts[0] != s $s\n" if ($bSts[0] != $s); \
          $bE = $bSts[0] + $bSzs[0]; \
          die "bE $bE != e $e\n" if ($bE != $e); \
          $bSts[0] -= $s; \
        } elsif ($bs == 2) { \
          $bSts[0]--;  $bSts[1]--; \
          if ($bSts[0] > $bSts[1]) { \
            # warn "Swapping $n ($bSts[0] > $bSts[1])\n"; \
            $tmp = $bSts[0];  $bSts[0] = $bSts[1];  $bSts[1] = $tmp; \
            $tmp = $bSzs[0];  $bSzs[0] = $bSzs[1];  $bSzs[1] = $tmp; \
          } \
          if ($bSts[0] != $s) { \
            # warn "Tweaking $n start from $s to $bSts[0]\n"; \
            $s = $bSts[0]; \
          } \
          $bE0 = $bSts[0] + $bSzs[0]; \
          $bE1 = $bSts[1] + $bSzs[1]; \
          $bE = $bE0 > $bE1 ? $bE0 : $bE1; \
          if ($bE != $e) { \
            # warn "Tweaking $n end from $e to $bE\n"; \
            $e = $bE; \
          } \
          $bSts[0] -= $s;  $bSts[1] -= $s; \
        } else { die "#blks is $bs for $n\n"; } \
        $bSt = join(",", @bSts) . ",";  $bSz = join(",", @bSzs) . ","; \
        $rgb = ($n =~ /deletion/) ? "224,0,0" : \
               ($n =~ /insertion/) ? "0,0,224" : \
               ($n =~ /inversion/) ? "0,224,0" : \
               ($n =~ /OEA/) ? "240,160,64" : "0,0,0"; \
        $_ = join("\t", $c, $s, $e, $n, $sc, $st, $s, $e, $rgb, \
                        $bs, $bSz, $bSt) . "\n";' $f \
      | hgLoadBed -tab hg17 $track stdin
    end
    perl -pe 'next if s/^track .*\n$//; \
        ($c, $s, $e, $n, $sc, $st, $tS, $tE, $r, $bs, $bSz, $bSt) = split; \
        @bSts = split(",", $bSt);  @bSzs = split(",", $bSz); \
        if ($n =~ /transchr/) { \
          $bs = 1; \
          $#bSts = 0;  $#bSzs = 0; \
        } elsif ($n =~ /OEA/) { \
          $s--; # weird that this is required only for OEA here \
          die "$n: bSts[0] $bSts[0] != 0\n" if ($bSts[0] != 0); \
          $bE = $s + $bSts[0] + $bSzs[0]; \
          die "$n: bE $bE != e $e\n" if ($bE != $e); \
        } elsif ($bs == 2) { \
          $bSts[0] += $s;  $bSts[1] += $s; \
          if ($bSts[0] > $bSts[1]) { \
            # warn "Swapping $n ($bSts[0] > $bSts[1])\n"; \
            $tmp = $bSts[0];  $bSts[0] = $bSts[1];  $bSts[1] = $tmp; \
            $tmp = $bSzs[0];  $bSzs[0] = $bSzs[1];  $bSzs[1] = $tmp; \
          } \
          if ($bSts[0] != $s) { \
            # warn "Tweaking $n start from $s to $bSts[0]\n"; \
            $s = $bSts[0]; \
          } \
          $bE0 = $bSts[0] + $bSzs[0]; \
          $bE1 = $bSts[1] + $bSzs[1]; \
          $bE = $bE0 > $bE1 ? $bE0 : $bE1; \
          if ($bE != $e) { \
            # warn "Tweaking $n end from $e to $bE\n"; \
            $e = $bE; \
          } \
         $bSts[0] -= $s;  $bSts[1] -= $s; \
        } else { die "#blks is $bs\n"; } \
        $bSt = join(",", @bSts) . ",";  $bSz = join(",", @bSzs) . ","; \
        $tS = $s;  $tE = $e; \
        $rgb = ($n =~ /deletion/) ? "224,0,0" : \
               ($n =~ /insertion/) ? "0,0,224" : \
               ($n =~ /inversion/) ? "0,224,0" : \
               ($n =~ /OEA/) ? "240,160,64" : "0,0,0"; \
        $_ = join("\t", $c, $s, $e, $n, $sc, $st, $tS, $tE, $rgb, \
                        $bs, $bSz, $bSt) . "\n";' G248discordant.txt \
    | hgLoadBed -tab hg17 kiddEichlerDiscG248 \
        stdin

    # 8/11/08: get clone ID -> NCBI acc mapping.
    ssh kkstore02
    mkdir /cluster/data/hg17/bed/kiddEichlerDiscordant/cloneIds
    cd /cluster/data/hg17/bed/kiddEichlerDiscordant/cloneIds
    # Saved off emailed file from Jeff Kidd to clones_used_3nov.txt.accessions;
    # get trace archive trace names for end reads:
    foreach n (7 9 10 11 12 13 14)
      wget http://hgsv.washington.edu/general/download/clone_mapping/ABC$n/ABC$n.conversion.gz
    end
    # ABC8 has _a and _b files:
    wget http://hgsv.washington.edu/general/download/clone_mapping/ABC8/ABC8_a.conversion.gz
    wget http://hgsv.washington.edu/general/download/clone_mapping/ABC8/ABC8_b.conversion.gz
    # That file is not available for G248.
    gunzip *.gz
    # Combine the relevant data from the .conversion files; keep only those
    # IDs that are used in the tracks.
    cut -f 4 ../bd35/*discordant.txt \
    | egrep -v '^(#chrom|track|name)' \
    | sed -e 's/,.*//' \
    | sort -u > discIds.txt
    perl -wpe 's/^OurClone.*\n// || s/^\d+_(HUMAN|\d+_).*\n$// || \
      s/^(\d+_)?(ABC|G)(\d+)(_\d\d?)?(_\d\d?)?_0*(\d+?_[A-Z]\d\d?)\.(F|FORWARD|R|REVERSE)\.\d+\t(\w+)$/$2$3_$6\t$7\t$8/ || \
      warn "Parse line $.:\n$_";' \
      *.conversion \
    | sort > allEnds.tab
    grep -wFf discIds.txt allEnds.tab > discEnds.txt
    wc -l discIds.txt allEnds.tab discEnds.txt 
#   223051 discIds.txt
# 17498527 allEnds.tab
#   573974 discEnds.txt
    # discEnds.txt has 2 lines (forward & reverse) for most of its ids...
    # ideally we would see 2*(223051) lines in discEnds.txt.
    # Get a list of which discordant clone IDs don't have ends in *.conv*:
    cut -f 1 allEnds.tab | uniq > all.tmp
    comm -23 discIds.txt all.tmp > discNotInConv.txt
    wc -l discNotInConv.txt
#16318 discNotInConv.txt
    cat > combine.pl <<'_EOF_'
#!/usr/bin/perl -w
use strict;
my ($cloneFile, $endsFile) = @ARGV;
open(CLONES, $cloneFile) || die "Can't open $cloneFile: $!\n";
my %idInfo;
while(<CLONES>) {
  (s/^(\d+_)?(ABC|G)(\d+)(_\d\d?)?(_\d\d?)?_0*(\d+?_[A-Z]\d\d?)\t(\w+)$/$2$3_$6\t$7/ && m/^(\w+)\t(\w+)/) || \
  m/^(G248\w+)\t(\w+)$/ || die "Parse line $.:$_";
  my ($id, $acc) = ($1, $2);
  $idInfo{$id}->[0] = $acc;
}
close(CLONES);
open(ENDS, $endsFile) || die "Can't open $endsFile: $!\n";
while (<ENDS>) {
  chomp; my ($id, $dir, $traceName) = split("\t");
  if ($dir =~ /^F/) {
    $idInfo{$id}->[1] = $traceName;
  } elsif ($dir =~ /^R/) {
    $idInfo{$id}->[2] = $traceName;
  } else { die "What is this \$dir: $dir ?\n"; }
}
close(ENDS);
foreach my $id (sort keys %idInfo) {
  my $infoRef = $idInfo{$id};
  $infoRef->[0] = '' if (! defined $infoRef->[0]);
  $infoRef->[1] = 0 if (! defined $infoRef->[1]);
  $infoRef->[2] = 0 if (! defined $infoRef->[2]);
  print join("\t", $id, @{$infoRef}) . "\n";
}
'_EOF_'
    # << emacs
    chmod a+x combine.pl
    combine.pl clones_used_3nov.txt.accessions discEnds.txt \
    | sort > kiddEichlerToNcbi.txt
    # Load table:
    ssh hgwdev
    cd /cluster/data/hg17/bed/kiddEichlerDiscordant/cloneIds
    hgLoadSqlTab hg17 kiddEichlerToNcbi \
      $HOME/kent/src/hg/lib/kiddEichlerToNcbi.sql kiddEichlerToNcbi.txt
    # Add to makeDb/schema/all.joiner, then check:
    runJoiner.csh hg17 kiddEichlerToNcbi $HOME/kent/src/hg/makeDb/schema


############################################################################
# TRANSMAP vertebrate.2008-05-20 build  (2008-05-24 markd)

vertebrate-wide transMap alignments were built  Tracks are created and loaded
by a single Makefile. This is available from:
   svn+ssh://hgwdev.cse.ucsc.edu/projects/compbio/usr/markd/svn/projs/transMap/tags/vertebrate.2008-05-20

see doc/builds.txt for specific details.
############################################################################


############################################################################
# KIDD/EICHLER VALIDATED SITES (DONE 6/11/08 angie)
    ssh hgwdev
    mkdir /cluster/data/hg17/bed/kiddEichlerValid
    cd /cluster/data/hg17/bed/kiddEichlerValid
    wget http://hgsv.washington.edu/general/download/validated_sites/Kidd_2008_sample_level_valided_sites.xls
    # Open in Excel, save as Kidd_2008_sample_level_valided_sites.txt,
    # move first 9 lines to Kidd_2008_sample_level_valided_sites.header.
    # Split into one file per individual:
    foreach id (Abc7 Abc8 Abc9 Abc10 Abc11 Abc12 Abc13 Abc14 G248)
      set ID = `echo $id | tr 'a-z' 'A-Z'`
      grep ${ID}_ Kidd_2008_sample_level_valided_sites.txt \
      | perl -wpe 'chomp; s/\r//; ($c, $s, $e, $n, $t) = split; \
        $rgb = ($n =~ /deletion/) ? "224,0,0" : \
               ($n =~ /insertion/) ? "0,0,224" : \
               ($n =~ /inversion/) ? "0,224,0" : "0,0,0"; \
        $t =~ s/:/,/g; \
        $n =~ s/^'$ID'_//;  $n = "$n,$t"; \
        $_ = join("\t", $c, $s, $e, $n, "0", "+", $s, $e, $rgb) . \
                  "\n";' \
      | hgLoadBed -tab hg17 kiddEichlerValid$id stdin
    end


################################################
# SPLIT EXPRESSION & REGULATION GROUPS
# (2008-09-09 kate)

echo "insert into grp (name, label, priority, defaultIsClosed) values ('expression', 'Expression', 4.5, 1)" | hgsql hg17
echo "update grp set label='Regulation' where name='regulation'" | hgsql hg17

################################################
# AUTOMATE UPSTREAM FILE CREATION (2008-10-15 markd)
update genbank.conf:
hg17.upstreamGeneTbl = refGene
hg17.upstreamMaf = multiz17way /hive/data/genomes/hg17/bed/multiz17way/species.lst


#############################################################################
# MAKE PCR TARGET FOR UCSC GENES (DONE 11/4/08 angie)
    ssh hgwdev
    mkdir /cluster/data/hg17/bed/mrnaPcr
    cd /cluster/data/hg17/bed/mrnaPcr
    # First, get consistent FA and PSL for UCSC Genes.
    genePredToBed /cluster/data/hg17/bed/kgHg17F/try3/kg3Try3.gp > ucscGenes.bed
    hgsql hg17 -NBe 'select kgId,geneSymbol from kgXref' \
    | perl -wpe 's/^(\S+)\t(\S+)/$1\t${1}__$2/ || die;' \
      > idSub.txt
    subColumn 4 ucscGenes.bed idSub.txt ucscGenesIdSubbed.bed
    sequenceForBed -keepName -db=hg17 -bedIn=ucscGenesIdSubbed.bed \
      -fastaOut=stdout \
    | faToTwoBit -ignoreDups stdin kgTargetSeq.2bit
    cut -f 1-10 /cluster/data/hg17/bed/kgHg17F/try3/kg3Try3.gp \
    | genePredToFakePsl hg17 stdin kgTargetAli.psl /dev/null

    # Load up the UCSC Genes target PSL table and put 2bit in /gbdb::
    cd /cluster/data/hg17/bed/mrnaPcr
    hgLoadPsl hg17 kgTargetAli.psl
    mkdir /gbdb/hg17/targetDb
    ln -s /cluster/data/hg17/bed/mrnaPcr/kgTargetSeq.2bit /gbdb/hg17/targetDb/

    # Ask cluster-admin to start an untranslated, -stepSize=5 gfServer on
    # /gbdb/hg17/targetDb/kgTargetSeq.2bit .

    ssh hgwdev
    # Add records to hgcentraltest blatServers and targetDb:
    hgsql hgcentraltest -e \
      'INSERT into blatServers values ("hg17Kg", "blat13", 17797, 0, 1);'
    hgsql hgcentraltest -e \
      'INSERT into targetDb values("hg17Kg", "UCSC Genes", \
         "hg17", "kgTargetAli", "", "", \
         "/gbdb/hg17/targetDb/kgTargetSeq.2bit", 1, now(), "");'


#############################################################################
# fox2ClipSeq from Gene Yeo (DONE - 2009-01-08 - Hiram)
    mkdir /hive/data/genomes/hg17/bed/fox2ClipSeq
    cd /hive/data/genomes/hg17/bed/fox2ClipSeq
    #	fetch data
    wget --timestamping \
'http://www.snl.salk.edu/~geneyeo/stuff/FOX2.rmsk.BED.gz' \
        -O FOX2.rmsk.BED.gz
    #	remove track line and sort
    zcat FOX2.rmsk.BED.gz | grep -v "^track" | sort -k1,1 -k2,2n \
        | gzip > sorted.bed.gz
    #	separate strand data, and turn the positive into blue
    zcat sorted.bed.gz | awk '$6 == "+"' | sed -e "s/255,0,0/0,0,255/" \
	| gzip > forwardStrand.bed.gz
    zcat sorted.bed.gz | awk '$6 == "-"' | gzip > reverseStrand.bed.gz
    #	turn into wiggle density plot
    zcat forwardStrand.bed.gz | bedItemOverlapCount hg17 stdin \
        | wigEncode stdin fox2ClipSeqDensityForwardStrand.wig \
	fox2ClipSeqDensityForwardStrand.wib
    #	Converted stdin, upper limit 2401.00, lower limit 1.00
    zcat reverseStrand.bed.gz | bedItemOverlapCount hg17 stdin \
        | wigEncode stdin fox2ClipSeqDensityReverseStrand.wig \
		fox2ClipSeqDensityReverseStrand.wib
    #	Converted stdin, upper limit 1406.00, lower limit 1.00
    #	and load tables
    zcat forwardStrand.bed.gz reverseStrand.bed.gz \
	| hgLoadBed hg17 fox2ClipSeq stdin
    #	Loaded 4418298 elements of size 9
    ln -s `pwd`/*.wib /gbdb/hg17/wib
    hgLoadWiggle hg17 fox2ClipSeqDensityForwardStrand \
	fox2ClipSeqDensityForwardStrand.wig
    hgLoadWiggle hg17 fox2ClipSeqDensityReverseStrand \
	fox2ClipSeqDensityReverseStrand.wig
    #	add composite track definitions to makeDb/trackDb/human/trackDb.ra

#############################################################################
# LIFTOVER TO Hg19 (DONE - 2009-04-24 - Hiram )
    mkdir /hive/data/genomes/hg17/bed/blat.hg19.2009-04-24
    cd /hive/data/genomes/hg17/bed/blat.hg19.2009-04-24
    # -debug run to create run dir, preview scripts...
    doSameSpeciesLiftOver.pl -debug hg17 hg19
    # Real run:
    time nice -n +19 doSameSpeciesLiftOver.pl -verbose=2 \
	-bigClusterHub=pk -dbHost=hgwdev -workhorse=hgwdev \
	 hg17 hg19 > do.log 2>&1 &
    #	real    84m19.022s

#############################################################################
############################################################################
# TRANSMAP vertebrate.2009-09-13 build  (2009-09-20 markd)

vertebrate-wide transMap alignments were built  Tracks are created and loaded
by a single Makefile. This is available from:
   svn+ssh://hgwdev.cse.ucsc.edu/projects/compbio/usr/markd/svn/projs/transMap/tags/vertebrate.2009-09-13

see doc/builds.txt for specific details.
############################################################################
# UPDATE KEGG TABLES (DONE, Fan, 6/18/10)

mkdir -p /hive/data/genomes/hg17/bed/pathways/kegg
cd /hive/data/genomes/hg17/bed/pathways/kegg

wget --timestamping ftp://ftp.genome.jp/pub/kegg/pathway/map_title.tab

cat map_title.tab | sed -e 's/\t/\thsa\t/' > j.tmp
cut -f 2 j.tmp >j.hsa
cut -f 1,3 j.tmp >j.1
paste j.hsa j.1 |sed -e 's/\t//' > keggMapDesc.tab
rm j.hsa j.1
rm j.tmp

hgsql hg17 -e 'drop table keggMapDesc'
hgsql hg17 < ~/kent/src/hg/lib/keggMapDesc.sql
hgsql hg17 -e 'load data local infile "keggMapDesc.tab" into table keggMapDesc'

wget --timestamping ftp://ftp.genome.jp/pub/kegg/genes/organisms/hsa/hsa_pathway.list

cat hsa_pathway.list| sed -e 's/path://'|sed -e 's/:/\t/' > j.tmp
hgsql hg17 -e 'drop table keggPathway'
hgsql hg17 < ~/kent/src/hg/lib/keggPathway.sql
hgsql hg17 -e 'load data local infile "j.tmp" into table keggPathway'

hgsql hg17 -N -e \
'select name, locusID, mapID from keggPathway p, knownToLocusLink l where p.locusID=l.value' \
>keggPathway.tab

hgsql hg17 -e 'delete from keggPathway'

hgsql hg17 -e 'load data local infile "keggPathway.tab" into table keggPathway'

rm j.tmp
############################################################################
# Add KEGG column to hg17 Gene Sorter (Done, Fan, 6/18/2010)

mkdir -p /hive/data/genomes/hg17/bed/geneSorter
cd /hive/data/genomes/hg17/bed/geneSorter
hgsql hg17 -N -e 'select kgId, mapID, mapID, "+", locusID from keggPathway' |sort -u|sed -e 's/\t+\t/+/' > knownToKeggEntrez.tab

hgsql hg17 -e 'drop table knownToKeggEntrez'

hgsql hg17 < ~/kent/src/hg/lib/knownToKeggEntrez.sql

hgsql hg17 -e 'load data local infile "knownToKeggEntrez.tab" into table knownToKeggEntrez'

#############################################################################
