#! /usr/bin/gawk -f
# Last edited on 2009-12-04 02:40:26 by stolfi

# Reads the file with raw but reformatted size of wikipedia
# at arbitrary dates, in the format
#   "{TIME} {YEAR} {MONTH} {DAY}  {SZ} {SZOK}"

# Outputs the same data in the format 
#   "{TIME} {YEAR} {MONTH} {DAY}  {SZ} {SU}  {DZ} {DU}"
# where {TIME} is spaced a fixed number {sper} of days apart starting with {sper-1}, {SZ} is size
# interpolated at {TIME}, {DZ} is the increment in {SZ} during the last {sper} days.
# The fields {SU} and {DU} are reliability flags (1 ok, 0 dubious, 9 ignore)

# Assumes that it was executed with "-f lib_date_time.gawk"

BEGIN{ 
  abort = -1;

  yz = 2001;   # Year zero of Wikipedia
  ott = -1;    # Time (days) for last OK input entry.
  osz = 0;     # Size as of last OK input entry, adjusted.
  
  oxsz = 0;    # Size field of last output entry.
  oxsu = 1;    # Status of size field of last output entry.
  
  sper = 28;       # Sampling period.
  xtt = sper-1;    # Time of next output entry.
  maxgap = 2*sper; # Distrust interpolations over more than these many days.
  nout = 0;        # Output data line count.

  # Parameters of articles to exclude from the count:
  
  split("", tosub); # {tosub[tt]} is the amount to subtract from day {tt} onwards.
  # Rambot articles:
  tosub[ 655] =    350 ;
  tosub[ 656] =   6690 ;
  tosub[ 657] =   5194 ;
  tosub[ 658] =    316 ;
  tosub[ 659] =   7217 ;
  tosub[ 660] =   2903 ;
  tosub[ 661] =   3880 ;
  tosub[ 662] =   6100 ;
  tosub[ 663] =    350 ;
  
  tosub_tot = 0;  # Cumulative {tosub} values up to present entry.

  # gawk \
  #    ' BEGIN{ss = 0;} 
  #      //{ 
  #        tt=$1; dy=$4; sz=$5; 
  #        sp = 54339 + (dy-17)/(25-17)*(54737 - 54339); 
  #        dp = sz - sp - ss; 
  #        printf "  tosub[%4d] = %6d;\n", tt, dp; 
  #        ss += dp; 
  #      } 
  #    '

  # Parameters of correction to compensate undercounting by {mpacIII}
  
  bCorr = 588;
  eCorr = 871;
  fCorr = 1.06;

  date_time_init()  
  # date_time_sanity_check(yz)  
}

(abort >= 0) { exit abort; }

/^ *([\#]|$)/ { print; next; } 

/^[ 0-9]*$/ { 
  if (NF != 6) { data_error(("bad line format"), lin); }
  tt = 0+$1; yr = 0+$2; mo = 0+$3; da = 0+$4; sz = 0+$5; su = 0+$6;
  check_entry(yz, tt, yr, mo, da, sz, su);
  if (su != 1) { next; }
  sz = adjust_raw_size(yz, tt, yr, mo, da, sz);
  process_entry(yz, tt, yr, mo, da, sz);
  next
}

// { data_error(("unrecognized line format"), $0); next; }

END{
  fflush();
  printf "subtracted %d articles overall\n", tosub_tot > "/dev/stderr";
}

function check_entry(yz,tt,yr,mo,da,sz,su,   r,xsz)
{
  if ((tt < 0) || (tt > 36500))
    { data_error(("bad time [" tt "]"), $0); }
  if (tt != time_from_date(yz, yr, mo, da))
    { data_error(("date error [" tt "]"), $0); }
  if ((yr < yz) || (yr > 2099))
    { data_error(("bad year [" yr "]"), $0); }
  if ((mo < 1) || (mo > 12))
    { data_error(("bad month [" mo "]"), $0); }
  if ((da < 1) || (da > 31))
    { data_error(("bad day [" da "]"), $0); }
  if ((sz < 0) || (sz > 999999999))
    { data_error(("bad size [" sz "]"), $0); }
  if ((su != 0) && (su != 1))
    { data_error(("bad status [" su "]"), $0); }
}

function adjust_raw_size(yz,tt,yr,mo,da,sz,   tk,sc)
{
  # Check chrono order
  if (tt <= ott) { data_error(("seq error [" ott "] [" tt "]"), $0); }
  
  # Gather all {tosub[tk]} entries since last entry to present:
  tk = ott+1;
  while (tk <= tt) { if (tk in tosub) { tosub_tot += tosub[tk]; } tk++; }
  
  # Subtract what is to be subtracted:
  sz = sz - tosub_tot;
  
  # Now aply {mpacIII} correction:
  if ((tt >= bCorr) && (tt <= eCorr)) { sz = int(sz*fCorr + 0.5); }
  
  return sz;

}

function process_entry(yz,tt,yr,mo,da,sz,   r,xsz,xsu,xdz,xdu,xdt,mtt)
{
  # Check chrono order
  if (tt <= ott) { data_error(("seq error [" ott "] [" tt "]"), $0); }
  if (xtt <= tt)
    {
      # Get sampling time {mtt} closest to the center of the interval:
      mtt = (tt + ott)/2;
      mtt = (int((mtt + 1)/sper) - 1)*sper + (sper - 1);
      if (mtt < xtt) { mtt = xtt; }
      if (mtt > tt)
        { data_error(("midsample bug [" ott "] [" xtt "] [" mtt "] [" tt "]"), $0); }
      # Generate interpolated data.
      while (xtt <= tt)
        { if (xtt < ott) { data_error(("sampling bug [" ott "] [" xtt "] [" tt "]"), $0); }

          # Interpolate geometrically the size {xsz} at time {xtt}:
          r = (xtt - ott + 0.0)/(tt - ott + 0.0);
          xsz = (osz+1)*exp(r*log((sz+1)/(osz+1))) - 1;

          # The interpolated value is reliable iff the gap is at most {maxgap} days:
          xsu = (tt - ott > maxgap ? 0 : 1);

          # Compute recent growth rate {xdz} in articles per sampling period:
          xdz = xsz - oxsz;

          # The growth rate is reliable iff the gap is at most {maxgap} days:
          xdu = (tt - ott > maxgap ? 0 : 1);
          # ... and if the previous size was reliable:
          if (oxsu != 1) { xdu = 0; }
          # ... and we are as close as possible to the gap's midpoint:
          if (xtt != mtt) { xdu = 9; }

          # Convert time of interpolated sample back to year, month, day:
          xdt = date_from_time(yz, xtt);
          # Print an explanatory comment:
          if (nout == 0) { printf "# Interpolated by compute-wp-growth-rate.gawk\n\n"; }
          printf "%6d %s %10d %d %10d %d\n", xtt, xdt, xsz, xsu, xdz, xdu; 
          oxsz = xsz;
          oxsu = xsu;
          xtt += sper;
          nout++;
        }
    }
  ott = tt; 
  osz = sz;
}

function data_error(msg, lin)
{ 
  printf "%s:%d: **data error - %s\n", FILENAME, FNR, msg > "/dev/stderr";
  printf " line = «%s»\n", lin > "/dev/stderr";
  abort = 1;
  exit 1;
}