#! /bin/bash -u
# Last edited on 2026-07-21 23:06:49 by stolfi

usage="$0 {CLEAN_DERIVED} {MAKE_WDS} {MAKE_TLW} {MAKE_KIND_TLW}"

clean_derived="$1"; shift;  # Remove all data files created by this note.

echo "GETTING THE SAMPLE SIZES FOR VOYNICHESE" 1>&2
echo " " 1>&2

# Get number of good tokens in Voynichese reference sample 
# (plain prose and labels):

vvers=( prs lab maj )
for book in "${vvers[@]}" ; do
  gfile="dat/voyn/${book}/tot.1/gud.wfr "
  cat ${gfile} \
    | gawk '/./{s+=$1} END{print s}' \
    > .tmp
  printf "${book} = %8d\n" `cat .tmp` 1>&2
done

echo "CHOOSING THE LEXEME SAMPLES FROM LANGUAGES OTHER THAN VOYNICHESE" 1>&2
echo " " 1>&2

# Gather the list ${smpsecs} of samples and sections ({lang}/{book}/{sec}),
# and the list ${smps} of samples without sections: 

echo "getting the list of samples and sections ..."
cat sample-sections.tbl \
  | gawk '/^ *([#]|$)/ { next; } // { print $1; }' \
  > .tmp
smpsecs=( `cat .tmp` )

echo "getting the list of samples ({lang}/{book}) ..."
echo ${smpsecs[@]} \
  | tr ' ' '\012' \
  | sed -e 's:[/][^/]*$::' \
  | uniq \
  > .tmp
smps=( `cat .tmp` )

echo "samples ({lang}/{book}/{sec}):" 1>&2
echo ${smps[@]} | tr ' ' '\012' | sed -e 's:^:    :g'

echo "sample sections ({lang}/{book}/{sec}):" 1>&2
echo ${smpsecs[@]} | tr ' ' '\012' | sed -e 's:^:    :g'

echo "creating per-sample \"sections.tags\" and \"sections-ok.tags\" ..." 1>&2
# Create files ${lang}/${book}/sections.tags and ${lang}/${book}/sections-ok.tags
# containing the list of sections (other than "tot.1") for each ${lang}/${book}.
# Also create links to the original langbank file "main.wds":

for smp in ${smps[@]} ; do
  lang="${smp/\/*/}"
  book="${smp/*\//}"
  
  if [[ ! ( -d dat/${lang}/${book} ) ]]; then mkdir -p dat/${lang}/${book}; fi
  if [[ ! ( -d tex/${lang}/${book} ) ]]; then mkdir -p tex/${lang}/${book}; fi
  sfile="dat/${lang}/${book}/sections.tags"
  sokfile="dat/${lang}/${book}/sections-ok.tags"
  cat sample-sections.tbl \
    | egrep -e '^ *'"${lang}/${book}/" \
    | gawk '// { s = $1; sub(/^.*[\/]/, "", s); print s; }' \
    | egrep -v -e '^tot[.]1$' \
    > ${sfile}
  echo "${lang}/${book} sections =    " `cat ${sfile} | tr '\012' ' '` 1>&2
  cp -p ${sfile} ${sokfile}
  echo "${lang}/${book} ok sections = " `cat ${sokfile} | tr '\012' ' '` 1>&2

  for sec in `cat ${sokfile}` tot.1 ; do
    smpsec="${lang}/${book}/${sec}"
    if [[ ! ( -d dat/${lang}/${book}/${sec} ) ]]; then mkdir -p dat/${lang}/${book}/${sec}; fi
    if [[ ! ( -d tex/${lang}/${book}/${sec} ) ]]; then mkdir -p tex/${lang}/${book}/${sec}; fi
  done
  
  if [[ ${clean_derived} -ne 0 ]]; then
    echo "REMOVING ALL DERIVED FILES" 1>&2
    for sec in  `cat ${sokfile}` tot.1 ; do
      for sizeopt in whole trunc; do 
        for kind in raw gud bad; do 
          make LANG=${lang} BOOK=${book} SEC=${sec} SIZEOPT=${sizeopt} KIND=${kind} clean
        done
      done
    done
  fi

  wdsFile="dat/${lang}/${book}/org/main.wds"
  make LANG=${lang} BOOK=${book} ${wdsFile}
  echo "checking if ${wdsFile} is ISO-Latin ..." 1>&2
  file ${wdsFile} | egrep -v -e 'ISO-8859|ASCII' > .noniso
  if [[ -s .noniso ]]; then
    echo "** not ISO-Latin:" 1>&2
    cat .noniso 1>&2
    exit 1
  fi

  for sec in `cat ${sokfile}` tot.1 ; do

    # We do two passes on the original text file, with {sizeopt} equal to
    # "whole" and "trunc", respectively.
    # 
    # In each pass, for for each sample and section {(lang"/"book"/"sec)???}
    # (including the pseudo-section "tot.1"), we create a token list
    # "dat/{(lang"/"book"/"sec)???}/{sizeopt}/raw.tlw" with the tokens from the text, one per
    # line. The "whole" version uses the full source text, while the "trunc"
    # version truncates the text to a prescribed number of "good" tokens (see
    # below).

    for sizeopt in whole trunc ; do
      tlwRawFile="dat/${lang}/${book}/${sec}/${sizeopt}/raw.tlw"
      make LANG=${lang} BOOK=${book} SEC=${sec} SIZEOPT=${sizeopt} ${tlwRawFile}

      # Split the file "raw.tlw" into "gud.tlw" and "bad.tlw":

      for kind in gud bad; do 
        tlwKindFile="dat/${lang}/${book}/${sec}/${sizeopt}/${kind}.tlw"
        make LANG=${lang} BOOK=${book} SEC=${sec} SIZEOPT=${sizeopt} ${tlwKindFile}
      done

      # For each {kind} in "raw", "gud", "bad", we also generate the files
      #   "{kind}.wdf" - tokens formated as running text.
      #   "{kind}.wfr" - occurrence counts and frequencies of lexemes.
      #   "{kind}-wds-summary.tex" - TeX macros with statistics.

      for kind in raw gud bad; do
        wdfFile="dat/${lang}/${book}/${sec}/${sizeopt}/${kind}.wdf"
        wfrFile="dat/${lang}/${book}/${sec}/${sizeopt}/${kind}.wfr"
        texFile="dat/${lang}/${book}/${sec}/${sizeopt}/${kind}-wds-summary.tex"
        make LANG=${lang} BOOK=${book} SEC=${sec} SIZEOPT=${sizeopt} KIND=${kind} \
          ${wdfFile} ${wfrFile} ${texFile}
      done
    done
  done
done

echo "summary of original \"main.wds\" files:" 1>&2
list_sample_wds_files.sh ${smps[@]}

echo "summary of raw \".tlw\" files:" 1>&2
for sizeopt in whole trunc ; do
  list_sizeopt_kind_tlw_files.sh ${sizeopt} raw ${smpsecs[@]}
done

echo "summary of good and bad token files:" 1>&2
for sizeopt in whole trunc ; do
  for kind in gud bad; do
    list_sizeopt_kind_tlw_files.sh ${sizeopt} ${kind} ${smpsecs[@]}
  done
done
 
# Finally, for each {sizeopt} and {kind} we create a global table 
# "{sizeopt}/{kind}-summary.txt" with basic data of all samples
# and sections.
echo "summary of good and bad token files:" 1>&2
for sizeopt in whole trunc ; do
  summarize_counts.sh ${sizeopt} ${smpsecs[@]}
done
for kind in raw gud bad ; do 
  printf "\n"
  paste summary-{whole,trunc}-${kind}.txt | expand | sed -e 's:^:    :g' 
done