#! /bin/bash -e
# Last edited on 2025-09-12 08:33:19 by stolfi

for sample in latn/ahl engl/ahe ; do
  for pos in 0 1 2 ; do
    ifile="langbank/${sample}/main.wds"
    echo "=== ${ifile} pos = ${pos} ==="
    extract_parag_words.gawk \
        -i work/error_funcs.gawk \
        -v sample_size=12 \
        -v sample_pos=${pos} \
        ${ifile} \
      > .wds
    echo "== words =="
    cat .wds \
      | sort | uniq -c \
      | sort -b -k1,1nr -k2,2 \
      | compute_freqs.gawk \
      > out/top-${pos}.wds
    head -n 10 out/top-${pos}.wds
    echo "== chars =="
    cat .wds \
      | sed -e 's:\(.\):\1@:g' \
      | tr '@' '\012' \
      | egrep -e '.' \
      | sort | uniq -c \
      | sort -b -k1,1nr -k2,2 \
      | compute_freqs.gawk \
      > out/top-${pos}.chars
    head -n 10 out/top-${pos}.chars
    echo "== char pairs =="
    cat .wds \
      | sed -e 's:\(.\):\1@\1:g' -e 's:^:.:g' -e 's:$:.:g' \
      | tr '@' '\012' \
      | egrep -e '.' \
      | sort | uniq -c \
      | sort -b -k1,1nr -k2,2 \
      | compute_freqs.gawk \
      > out/top-${pos}.pairs
    head -n 10 out/top-${pos}.pairs
  done
done