#! /bin/bash
# Last edited on 2009-12-05 17:27:11 by stolfi

# Processing, analyzing, and modeling the Wikipedia article count growth.

# Get the phase to start at:
if [[ $# -gt 0 ]]; then
  ini="$1"; shift
else
  ini=0
fi
if [[ $# -gt 0 ]]; then
  fin="$1"; shift
else
  fin=999
fi
  
if [[ ( ${ini} -le 1 ) && ( ${fin} -ge 1 ) ]]; then
# REFORMATTING THE RAW DATA FILES
# 
#   Reformat the files to put them in a common format
#   
#      "{TIME} {YEAR} {MONTH} {DAY}  {SZ} {SU}"
#      
#   where {YEAR} is the 4-digit year, {MONTH} is 01 to 12, {DAY} is day
#   of month 01 to 31, {TIME} is elapsed days since Jan 1, 2001, {SZ} is
#   the article count at {TIME}, and {SU} is a status indicator, 
#   presently 1 (OK) or 0 (suspicious).
# 
  
    for ff in 2009-06 ; do
      cat data/wp-size-emo-${ff}-raw.txt \
        | reformat-wp-size-emo-raw.gawk -f lib_date_time.gawk \
        > rslt/wp-size-emo-${ff}-rar.txt
    done
    
# We must exclude a very wrong line with duplicated date
  
    for ff in 2009-11 ; do
      cat data/wp-size-irr-${ff}-raw.txt \
        | egrep -v -e 'RAMBOT causing Error' \
        | reformat-wp-size-irr-raw.gawk -f lib_date_time.gawk \
        > rslt/wp-size-irr-${ff}-rar.txt
    done

#   Plotting the reformatted raw data:
  
    for nodub in 0 1 ; do
      for early in 0 1 ; do
        for ff in emo-2009-06 irr-2009-11 ; do
          plot-wp-size-rar.sh \
              ${early} \
              ${nodub} \
              rslt/wp-size-${ff}-rar.txt \
            > rslt/wp-size-${ff}-rar-e${early}-n${nodub}.eps
        done
      done
    done
fi

if [[ (${ini} -le 2 ) && ( ${fin} -ge 2 ) ]]; then
# PLOTTING AROUND THE MAJOR GLITCHES

    plot-counter-glitch.sh \
       -13750 14930 0.0644 \
        19250 14930 0.0644 \
        data/counter-glitch-2002-10-irr.txt \
      > rslt/counter-glitch-2002-10-irr.eps
     
    plot-counter-glitch.sh \
        31378 12126 0.0644 \
        33261 12853 0.0644 \
        data/counter-glitch-2003-05-irr.txt \
      > rslt/counter-glitch-2003-05-irr.eps

fi

# Only the irr-2009-11 dataset is worth processing:
datasets=( irr-2009-11 )
# datasets=( irr-2009-11 emo-2009-06 )

if [[ ( ${ini} -le 3 ) && ( ${fin} -ge 3 ) ]]; then
# COMPUTING THE GROWTH RATES
# 
#   Converting to the format "{TIME} {YEAR} {MONTH} {DAY}  {SZ} {SU}  {DZ} {DU} "
#   where {TIME} is spaced a fixed number {sper} of days apart starting with {sper-1}, {SZ} is size
#   interpolated at {TIME}, {DZ} is the increment in {SZ} during the last {sper} days.
#   The flag {SU} is as as before. The flag {DU} is 1 (OK), 0 (suspicious) or 9 (don't use).
  
    for ff in ${datasets[@]} ; do
      cat rslt/wp-size-${ff}-rar.txt \
        | compute-wp-growth-rate.gawk -f lib_date_time.gawk \
        > rslt/wp-size-${ff}-int.txt
      for early in 0 1 ; do
        plot-wp-growth-rate.sh \
            ${early} \
            rslt/wp-size-${ff}-int.txt \
          > rslt/wp-size-${ff}-int-e${early}.eps
        plot-raw-and-fixed-size.sh \
            ${early} \
            rslt/wp-size-${ff}-rar.txt \
            rslt/wp-size-${ff}-int.txt \
          > rslt/wp-size-${ff}-rar-int-e${early}.eps
      done
    done
fi

if [[ ( ${ini} -le 4 ) && ( ${fin} -ge 4 ) ]]; then
# MODELING
  
    for ff in ${datasets[@]} ; do
      for mod in 000 010 011 ; do
        echo "===============================================================" 1>&2 
        echo "dataset = ${ff} model = ${mod}" 1>&2 
        for sea in 0 1 ; do
          for pha in 0 1 2 ; do
            cat rslt/wp-size-${ff}-int.txt \
              | predict-wp-size.gawk \
                  -f lib_date_time.gawk \
                  -f lib_functions.gawk \
                  -f model_params_${mod}.gawk \
                  -v phase=${pha} \
                  -v seasonal=${sea} \
              > rslt/wp-size-${ff}-prd-p${pha}-s${sea}-m${mod}.txt
          done
        done

        for early in 0 1 ; do
          for sea in 0 1 ; do
            echo "early = ${early} seasonal = ${sea}" 1>&2
            for pha in 0 1 2 ; do
              if [[ ( ${early} -gt 0 ) && ( ${pha} -gt 0 ) ]]; then continue; fi
              for logscale in 1 0 ; do
                if [[ ( ${logscale} -gt 0 ) && ( ${early} -gt 0 ) ]]; then continue; fi
                plot-predicted-growth.sh \
                    ${early} \
                    ${logscale} \
                    rslt/wp-size-${ff}-prd-p${pha}-s${sea}-m${mod}.txt \
                  > rslt/wp-size-${ff}-prd-p${pha}-s${sea}-m${mod}-dz-e${early}-y${logscale}.eps
                plot-predicted-growth-error.sh \
                    ${early} \
                    ${logscale} \
                    rslt/wp-size-${ff}-prd-p${pha}-s${sea}-m${mod}.txt \
                  > rslt/wp-size-${ff}-prd-p${pha}-s${sea}-m${mod}-de-e${early}.eps
                if [[ ${pha} -gt 0 ]]; then continue; fi
                plot-predicted-size.sh \
                    ${early} \
                    ${logscale} \
                    rslt/wp-size-${ff}-prd-p${pha}-s${sea}-m${mod}.txt \
                  > rslt/wp-size-${ff}-prd-p${pha}-s${sea}-m${mod}-sz-e${early}-y${logscale}.eps
                plot-predicted-size-error.sh \
                    ${early} \
                    ${logscale} \
                    rslt/wp-size-${ff}-prd-p${pha}-s${sea}-m${mod}.txt \
                  > rslt/wp-size-${ff}-prd-p${pha}-s${sea}-m${mod}-se-e${early}.eps
              done
            done
          done
        done
      done
    done
fi

if [[ ( ${ini} -le 5 ) && ( ${fin} -ge 5 ) ]]; then
# OMEGA AND SIGMA

  plot-omega.sh > rslt/omega.eps
  plot-sigma.sh > rslt/sigma.eps
  
fi