#!/usr/bin/env perl
#   $Id: wash.pl 19532 2007-10-25 00:11:48Z bjn $

# 2001-01-23 Willy Bertiger postbreak is now executed before edptn2
#                           hooks are in to have a postbreak after edptn2
#                           For gol2 test, 1999-10-28, static position
#                           differences with FLINN data base: 8, -2, 0 mm
#                           xyz, CHAMP tested for aug 6, 7 2000 also yielded
#                           good results

#my $Purpose = "
#A perl replacement for csh wash scripts.";
#

use 5.6.0;
use strict;
use warnings;
use Carp;

use lib "$ENV{GOA}/lib/perl";
use GOA::PathFinder;
use Stats;

print STDERR "Don't forget that this wash script modifies the rgfile\n";

my $cmd_line = join(" \\\n      ", $0, @ARGV);
write_log($cmd_line);
print "     Using $ENV{SMSOLIN} for smapper iteration\n" if($ENV{"SMSOLIN"});

my $tdp_in = "";             # nominal tdp for smoother
my $dtype  = "dual";         # indicates dual or single freq. GPS data
                             # currently just used to set postbreak namelist

my $leo_oi = "";             # filter needs the leo oi file for reduced 
                             # dynamic runs; if set modifies the filter
                             # command below
my @post_wind=(10E-3,200E-5);# final outlier detection window for range
                             # and phase (km); for high quality data
                             # (5E-3, 50E-5) 

my $pb_min_slip = 20E-5;     # for postbreak
my $pb_max_iter = 2;         # max iterations for postbreak
my $flag_brks_qm = 0;        # if true mark qmfile with new breaks
my $flag_qm = 0;             # flag postfit outliers

my $edtpnt_max = 6;          # edtpnt2 cycles
my $devf = "";               # dev version of filter, default OFFICIAL
my $devsm = "";              # dev version of smapper, default OFFICIAL



# if there was a previous reduced dynamic run ( interation on dynamic 
# stochastics the environment variable SMSOLIN should be set to 
# the previous smsol.nio output file. *****NOTE**** do not call
# it smsol.nio, since this file is removed at the beginning of the 
# script

while ( @ARGV ){             # command line argument options
     if       ($ARGV[0] =~ /^-tdp_in$/)   {shift;$tdp_in         = shift}
     elsif    ($ARGV[0] =~ /^-dtype$/)    {shift;$dtype          = shift}
     elsif    ($ARGV[0] =~ /^-oi$/)       {shift;$leo_oi         = shift}
     elsif    ($ARGV[0] =~ /^-post_wind$/)  {shift;@post_wind = (shift,shift)}
     elsif    ($ARGV[0] =~ /^-flag_brks_qm$/){shift;$flag_brks_qm  = 1}
     elsif    ($ARGV[0] =~ /^-flag_qm$/)     {shift;$flag_qm  = 1}
     elsif ($ARGV[0] =~ /^-edtpnt_max$/) {shift;$edtpnt_max =  shift}
     elsif ($ARGV[0] =~ /^-pb_min_slip$/){shift;$pb_min_slip = shift}
     elsif ($ARGV[0] =~ /^-devf$/) {shift; $devf  = "dev"}
     elsif ($ARGV[0] =~ /^-devsm$/){shift; $devsm = "dev" }
     else {die "Unkown argument: $ARGV[0]\n";}
}

my @post_wind_bias_brk= ( ( 20E-3 > $post_wind[0]? 20E-3:$post_wind[0] ),
                          ( 20E-3 > $post_wind[1]? 20E-3:$post_wind[1] ) ); 

                                     # outlier level for postfit before 
                                     # first bias break search

no warnings;
die "Environment variable SMSOLIN set to smsol.nio which will be removed. 
    You must use a different name" if($ENV{SMSOLIN} eq "smsol.nio");
use warnings;

#file names used by preprefilter, prefilter, filter smapper, posbreak
# assumes a qmfile with the name "qmfile" which will be modified if
# $flag_brks_qm with additional breaks from postbreak analysis
# if this option is used a temporary qmfile will be made

unlink qw(
       prefilter.txt
       batch.txt
       accume.nio
       smooth.nio
       uinv.nio
       smsol.nio
       smcov.nio
       point.txt
       postfit.nio
       APVALSOUT
       TDPTABLE 
       );

# Commands that are used multiple times


my $filter_cmd = "filter batch.txt wash.nml \'$leo_oi\' rgfile accume.nio ".
        "smooth.nio uinv.nio ''  $devf";

my $smapper_cmd = "smapper wash.nml '' accume.nio smooth.nio uinv.nio ".
    "smsol.nio smcov.nio '' $devsm";

my $postfit_cmd = "postfit wash.nml rgfile smsol.nio uinv.nio postfit.nio ".
    "point.txt";

my $flag_qm_cmd = "flag_qm -p point.txt -i qmfile_old -o qmfile ";

run_shell_cmd( "preprefilter wash.nml rgfile prefilter.txt" );
run_shell_cmd( "prefilter    prefilter.txt rgfile batch.txt" );

run_shell_cmd($filter_cmd);

$ENV{EPHFIL} = "$ENV{GOA_EPH}/$ENV{ARCH}/de405s.nio";
$ENV{TDPTABLE} = "TDPTABLE";
if ( $tdp_in ){
    print "Setting environment variable: TDPTABLEIN to $tdp_in\n".
        "for input to smapper\n";
    $ENV{TDPTABLEIN} = $tdp_in; 
} # smoother nominals

run_shell_cmd($smapper_cmd);

# prepare for postbreak analysis cycle
unlink( "postfit.nio" );
change_postfit_window(@post_wind_bias_brk );
run_shell_cmd($postfit_cmd);
if( $flag_qm && (-e "point.txt")){
    rename "qmfile", "qmfile_old" || die "Can't rename qmfile for flag_qm";
    run_shell_cmd($flag_qm_cmd);
}
unlink glob('batch.out.*');
my $postbreak_log = "postbreak.log";
unlink glob("$postbreak_log.*");
unlink("point.txt");            # postbreak doesn't use it

postbreak_nml_mk($dtype, $pb_min_slip, 200E-3); # max auto change 200 m

postbreak_cycle($pb_max_iter, $flag_brks_qm, $flag_qm, $postbreak_log );

# end postbreak analysis

# start looking for outliers 
my @range_window = (200, 20, 5*$post_wind[0], 4*$post_wind[0], 
                             2*$post_wind[0], $post_wind[0]);
my @phase_window = (200, 20, 5*$post_wind[1], 4*$post_wind[1], 
                             2*$post_wind[1], $post_wind[1]);
for( my $i = 0; $i <= $#range_window; $i++){
    print "Starting postfit Cycle: $i\n";
    unlink qw( postfit.nio point.txt);
    change_postfit_window( $range_window[$i], $phase_window[$i]);
    run_shell_cmd($postfit_cmd);
 
    if( $flag_qm && (-e "point.txt")){
        rename "qmfile", "qmfile_old" || die 
            "Can't rename qmfile to qmfile_old for flag_qm";
        run_shell_cmd($flag_qm_cmd);
    }
    
    if ( -e "point.txt"){
        edtpnt2_cycle($edtpnt_max, $flag_qm,
                       $range_window[$i], $phase_window[$i],$post_wind[0],$post_wind[1]);
    }

}

postbreak_nml_mk($dtype,$pb_min_slip,$pb_min_slip ); # auto will have no effect

unlink glob('batch.out.*');
$postbreak_log = "postbreak_postEdit.log";
unlink glob("$postbreak_log.*");
unlink("point.txt");            # postbreak doesn't use it
postbreak_cycle($pb_max_iter, $flag_brks_qm, $flag_qm, $postbreak_log);

rename "TDPTABLE" , "tdp_final"; # last was final

exit;


#################subroutines
# Bareword "CMDLOG" not allowed while "strict subs" in use
# Bareword "STDOUT" not allowed while "strict subs" in use
sub run_shell_cmd{              # one required arg, 2 optional
    my $cmd = shift;            # command and arguments to run
    my $die = 1; 
       $die = shift if (scalar @_ );     # if non-zero will not exit on 
                                         # non-zero exit status from $cmd
                                         # allows calling routine to check 
                                         # exit status and take action
    my $log = "";
       $log = shift if (scalar @_ );     # optional log file
    my $LOG;

    local (*CMDLOG);

    write_log( "\nStarting $cmd" );   
    my @cmd = split(' ', $cmd );
    if ( $log ) { 
        open( CMDLOG, ">$log") || die "unable to open $log file";
        print "   Logfile for this command: $log\n";
        $LOG = \*CMDLOG;
    }else{
        $LOG = \*STDOUT;
    }
    print $LOG `$cmd`;
    die "Fatal error in $cmd" if ($? && $die);
    close $LOG if $log;
    write_log( "Finished $cmd[0]\n");

}

sub write_log{
    my $s = shift;
    print scalar localtime," :\n   $s\n";
}

sub postbreak_nml_mk{
# Input:
    my $dtype = shift;          # Input data type single or dual
    my $pb_min_slip = shift;    # Input min slip ( auto is on )
                                # output file postbreak.nml in pwd
    my $pb_auto_slip = shift;   # max change with auto
                                # set to $pb_min_slip if < $pb_min_slip

    if($pb_auto_slip < $pb_min_slip){
        $pb_auto_slip = $pb_min_slip;
    }

    open( PNML, ">postbreak.nml") || die "Unable to open postbreak.nml";
    if ( $dtype eq "single" ){
        print PNML 
            " \$options\n dtyp = 121\n min_slip_size = $pb_min_slip\n".
            "  MIN_ARC = 5\n  DELTA = 5E-6\n  auto=.true.\n".
            "  max_auto_slip=$pb_auto_slip\n \$end\n";
    }else{
         print PNML 
            " \$options\n dtyp = 120\n min_slip_size = $pb_min_slip\n".
            "  MIN_ARC = 5\n  DELTA = 5E-6\n  AUTO=.TRUE.\n".
            "  max_auto_slip=$pb_auto_slip\n \$end\n";
     }
    close PNML;

}

sub change_postfit_window{
    my $r_win = shift;          # range window km
    my $faz_win = shift;        # phase window km

    my $e = "s/^\\s+WINDOW\\s+=\\s+\\d.*/  WINDOW = $r_win, $faz_win/";
                                # note the \\ above
    `perl -i -pe '$e' wash.nml`;

    print 
        "Postfit edit window changed to Range: $r_win Phase: $faz_win km\n";
}

sub edtpnt2_cycle{            # depends on global file names
                              # window in wash.nml should have been set first
    my $edtpnt_max = shift;
    my $flag_qm    = shift;
    my ($range_window_cur, $phase_window_cur,$post_wind_rng_fin,$post_wind_phs_fin ) = @_;
    my $ec = 0;                    # counter for edtpnt2 cycles
    
    unlink "EDIT_POINT_FAILURE" if( -e "EDIT_POINT_FAILURE" );
    while ( -e "point.txt"){       # edit point cycle

        if ( $ec == $edtpnt_max ){
            print "edit point cycle failed to converge after $ec cycles,".
                "THERE ARE PROBABLY LARGE OUTLIERS LEFT\n".
                "Window Range, Phase(km): $range_window_cur, $phase_window_cur\n".
                "Final Window(km): $post_wind_rng_fin,$post_wind_phs_fin\n";
            carp "edit point cycle failed to converge after $ec cycle.".
                " THERE ARE PROBABLY LARGE OUTLIERS LEFT \n".
                "Window Range, Phase(km): $range_window_cur, $phase_window_cur\n".
                "Final Window(km): $post_wind_rng_fin,$post_wind_phs_fin\n";

            if( $range_window_cur == $post_wind_rng_fin && $phase_window_cur == $post_wind_phs_fin){
                carp "\nedit point cycle failed to converge after $ec cycle at the FINAL EDIT stage\n";
                open( S, ">EDIT_POINT_FAILURE");
                print S "At the final edit point window range, phase(km): $post_wind_rng_fin,$post_wind_phs_fin, \n".
                    "the edit point cycle failed to converge and there may still be outliers in the solution.\n".
                    "Caution should be exercised.\n".
                    "Some change in residuals may occur after the remaining postbreak cycle.\n";
                print S "\n\nStatistics on remaining outliers:\n"; 
                my @out_stats = outlier_res($post_wind_phs_fin,$post_wind_rng_fin);
                printf S "Number_Range: %d RMS_range(km): %.15g (RMS_range-Window)/Window: %.15g\n", 
                @out_stats[3..5];
                printf S "Number_Phase: %d RMS_Phase(km): %.15g (RMS_Phase-Window)/Window: %.15g\n", 
                @out_stats[0..2];
                close S;
            }
            return;
        }

        print "------ Starting Edit Point Cycle(edtpnt2) iteration: $ec\n";
        run_shell_cmd("edtpnt2 point.txt wash.nml ".
                      "rgfile accume.nio smooth.nio uinv.nio", 0 );

        if($?){                     # bad exit from edtpnt2 re-run filter
            printf "edtpnt2 exit status: %d  RE-RUNNING FILTER\n", $?>>8;
            print "Exit 9 indicates too many points for edpnt2 to delete.\n";
            print "Re-running the filter on the marked regres file for exit other\n";
            print "than 9 could be trouble, Alpha, A near 1 probably OK with filter\n";
            print " re-run.\n";
            printf STDERR "edtpnt2 exit status: %d  RE-RUNNING FILTER\n", $?>>8;
            print STDERR 
                "Exit 9 indicates too many points for edpnt2 to delete.\n";
            print STDERR " To check look for \"TOO MANY DATA POINTS TO EDIT\"\n";
            print STDERR " Other exits may not be OK. \n";
            print STDERR " \"CAN NOT REMOVE THIS DATA POINT\" seems to be OK with ".
                " a filter re-run.\n";
            print STDERR "Re-running the filter on the marked regres file\n";
            unlink ("accume.nio","smooth.nio","uinv.nio");
            run_shell_cmd($filter_cmd);
        }                           # endif bad exit from edtpnt2

        unlink qw(point.txt smsol.nio smcov.nio postfit.nio TDPTABLE APVALSOUT);

        run_shell_cmd($smapper_cmd);
        run_shell_cmd($postfit_cmd);
        if( $flag_qm && (-e "point.txt")){
            rename "qmfile", "qmfile_old" || die 
                "Can't rename qmfile to qmfile_old for flag_qm";
            run_shell_cmd($flag_qm_cmd);
        }

        $ec++;                      # increment loop counter


    }                               # end edit point cycle

}

sub postbreak_cycle{

    my $pb_max_iter = shift;   # 1 will give one iteration
    my $flag_brks_qm = shift;   # modify qmfile if true with new breaks
    my $flag_qm      = shift;   # flag outliers from point.txt if true
    my $log_root     = shift;   # root name for log files

    my $post_brk_cmd = 
        "postbreak -p postfit.nio -n postbreak.nml -b batch.txt ";

    $post_brk_cmd .= "-qi qmfile_old -qo qmfile " if($flag_brks_qm);

#   Assumes a filter/smapper/postfit run before this
#   and namelist has been made (postbreak_nml_mk)
#   makes batch.out.* and $log_root.* 
#   at each stage cp batch.out.* to batch.out

    my $new_slips = 1;              # status returned by grep -q checking for 
                                    # new slips, changed to perl script
    my $pc = 0;                     # postbreak counter
    while( $new_slips ){
        if( $flag_brks_qm ){
            rename "qmfile", "qmfile_old" 
                || die "Can't rename old qmfile for postbreak";
        }
        run_shell_cmd("$post_brk_cmd -o batch.out.$pc", 1, "$log_root.$pc");
        `perl -ne 'exit 1 if eof; exit 0 if(/No new slips/)' $log_root.$pc`;
        $new_slips = $?;
        `cp  batch.out.$pc batch.txt`;
        print STDERR "Postbreak Analysis Failed to Converge $pb_max_iter iterations\n" 
            if ( $pc == $pb_max_iter-1 );

        $pc++;

        if ( $new_slips ){          # re-run filter smoother postfit
            unlink qw( accume.nio smooth.nio uinv.nio smsol.nio smcov.nio
                       point.txt postfit.nio TDPTABLE APVALSOUT );
            run_shell_cmd($filter_cmd);
            run_shell_cmd($smapper_cmd);
            run_shell_cmd($postfit_cmd);
            if( $flag_qm && (-e "point.txt")){
                rename "qmfile", "qmfile_old" || die 
                    "Can't rename qmfile to qmfile_old for flag_qm";
                run_shell_cmd($flag_qm_cmd);
            }
        }

        last if ($pc == $pb_max_iter ); 
    }                               # end postbreak analysis for cycle slips
# maybe addd option to remove unlink glob('batch.out.*'); unlink glob('postbreak.log.*');

}

sub outlier_res{

    my ($post_wind_phs_fin,$post_wind_rng_fin) = @_; # km dump residuals anbd look


    my $post = "postfit.nio";
    my $dev  = "";
    my $out = "residuals.txt$$";
    my $pltrej = ".FALSE.";

    my $phs_stats = new Stats;
    my $rng_stats = new Stats;

    die "No file: $post" if( ! -e "$post" );
    open( NML, "> rdposfit_dump.nml$$" ) || die "Unable to open file: ".
    " rdposfit_dump.nml$$ for temporary namelist";

    print NML "

 \$OPTIONS
  PLTREJ = $pltrej
  DTYP = 120, 110,111,121
  SATS = 'All'
  STNS = 'All'
  PRINT_AZ_EL_BOTH = .TRUE.
 \$END
";

    close NML;

    if( -e $out ){
        print "removing pre-existing $out\n";
        unlink $out;
    }

    `rdpostfit_dump rdposfit_dump.nml$$ $post $out $dev`;

    die "Error in rdpostfit_dump; exiting without removing temp files\n" if ($?);

    unlink "rdposfit_dump.nml$$";

    open( R, "<$out");
    while(<R>){
        my @l = split(' ');
        if( ($l[3] == 120 or $l[3] == 121) && abs($l[4]*1e-5) > $post_wind_phs_fin ){
            $phs_stats->add($l[4]*1e-5);
        }
        if( ($l[3] == 110 or $l[3] == 111) && abs($l[4]*1e-5) > $post_wind_rng_fin ){
            $rng_stats->add($l[4]*1e-5);
        }
    }
    unlink $out;

    my ($n_r,$rms_r, $n_p, $rms_p) = (($rng_stats->calc)[0,5], ($phs_stats->calc)[0,5]);
    return ($n_p, $rms_p, ($rms_p-$post_wind_phs_fin)/$post_wind_phs_fin, 
            $n_r, $rms_r, ($rms_r-$post_wind_rng_fin)/$post_wind_rng_fin);

}

__END__

=head1 NAME

wash.pl 

=head1 SYNOPSIS

Runs cycles of filter, smoother, postfit, postbreak, edpnt2, postbreak. 
Read the script for options. Note that postbreak cycles are run twice. 
Once before data editing using the postbreak auto feature limited to 
a max residual jump of 200 m and then another cycle after the edpnt2 
cycle without the auto option ( default slip of 20 cm or user supplied 
value).

If edpnt2 fails to converge, a file, EDIT_POINT_FAILURE, with diagnostic
information is created. 

=head1 AUTHOR

Willy Bertiger <willy.bertiger@jpl.nasa.gov>

=head1 COPYRIGHT

Copyright (c) 2003

Jet Propulsion Laboratory, California Institute of Technology

=cut
