#! /usr/bin/python3 # Last edited on 2026-07-01 13:00:39 by stolfi # Functions for measuring size and listing word position in a # parag. To be included in other python scripts. import sys, os, re from sys import stdout as out, stderr as err, stdin as inp from error_funcs import arg_error, file_line_error, prog_error from process_funcs import bash, basic_line_loop from chinese_funcs import read_chinese_char_set from note_077_funcs import compute_and_print_stats from math import sqrt, hypot, exp, log, pi, inf, nan, floor, ceil def clean_up_raw_text(text, utype, data_error): # Cleans up the raw {text} from an SBJ or SPS transcription file according to {utype} # for use in Note 077 analysis. # # See {clean_up_bencao_raw_text} for the meaning of {utype} "ch". # See {clean_up_starps_raw_text} for the meaning of {utype}s "ec", "wp", and "wc". # if utype == "ch": text = clean_up_bencao_raw_text(text, utype, data_error) elif utype == "ec" or utype == "wc" or utype == "wp": text, head, tail = clean_up_starps_raw_text(text, utype, data_error) return text # ---------------------------------------------------------------------- def normalize_text(text, utype, data_error): # Modifies a {text} (typically after {cleanup_text}) from an SBJ or SPS transcription # file according to {utype} for use in Note 077 analysis, by replacing # certain substrings that are assumed to be scribal errors, # abbreviations, or non-significant variants. # # See {normalize_bencao_text} for the meaning of {utype} "ch". # See {normalize_starps_text} for the meaning of {utype}s "ec", "wp", and "wc". # if utype == "ch": text = normalize_bencao_text(text, utype, data_error) elif utype == "ec" or utype == "wc" or utype == "wp": text = normalize_starps_text(text, utype, data_error) return text # ---------------------------------------------------------------------- def clean_up_starps_raw_text(text, utype, data_error): # Cleans up the raw {text} from an SPS transcription file according to {utype} # for use in Note 077 analysis. # # The function returns the cleaned {text} and booleans {head} and # {tail} that specify whether the line was a parag head line or parag # tail line, respectively. Note that both may be true for a one-line # parag. # # INPUT TEXT EXPECTED # # The raw {text} given must be a non-empty string that may contain # any of these: # # Prefix "<%>" marking the line as the head of a paragraph. # # Prefix [«=»] to indicate the alignment of the start of the line # relative to the left rail. # # EVA letters [A_Za-z]. # # Invalid EVA code '?'. # # Weirdo codes "&{NNN};" (possibly without the ';') # where {NNN} is three decimal digits. # # Ligatures consisting of two or more EVA letters, '?', or # weirdo codes enclosed in braces "{...}". # # Word separators [-.,]. # # Suffix [«=»] to indicate the alignment of the end of the line # relative to the right rail. # # Suffix "<$>" to mark the line as the tail of a parag. # # Inline comments "", including various special comments # to indicate stars in the margin, wide linegaps, figure intrusions. # vellum folds, etc. These may appear anywhere in the line # including before or after all the {text} and inside other markup # tags. # # Example: # # "={Ch}eeo,daiin.{Sh}eedy&162&211am.{Ch}odal.ol{Ch}edy«<$>" # "<%>=w{Ch}edairs.oeail.{Ch}otaropdaiin.otol,dair.aiir,aim=" # "={Ch}ol.oeedy.keedy.{Ch}{Ch}eky{air.ar.{Sh}ol.{Ch}edy.ot{Ch}edy.{Qo}ty=" # # Note the parag break between the first and second text. # # CLEANUP PERFORMED # # For all {utype}s, the cleanup entails: # # Removing all inline comments. # # Removing all markup, including the braces [{}], parag markers "<%>" and "<$>", # and rail alignment markers [«=»]. # # Mapping 'w' and 'z', used in my transcription to encode hooked puffs, # to 'p' and 'f', respctively. # # Mapping the uncertain reading charaters [bj] to '?'. # # Mapping all weirdo codes to '?'. # # Mapping everything to lowercase. # # Moreover, # # If {utype} is "ec", all punctuation [-,.] is deleted. # # If {utype} is "wp", every ',' is deleted, and '-' is mapped to '.' # # If {utype} is "wc", every ',' and '-' are mapped to '.' # # CLEANED TEXT # # After this cleanup, if {utype} is "ec", the text must have only # characters [ac-fhik-tvxy?]. Example: # # "choloeedykeedychchekycheod?eyke??edeedydaiiinaldair" # # If {utype} is "wc" or "wp", the raw text may also have '.' separators. Example: # # "chol.oeedy.keedy.chcheky.cheod?ey.ke??edeedy.daiiin.ald.air" # # For more restrictions, see {check_starps_cleanup} below. # text = text.strip() if utype == "ec" or utype == "wc" or utype == "wp": # Remove inline comments: text = re.sub(r"[<][!][^<>]*[>]", "", text) # Map weirdo codes to '?': text = re.sub(r"[&][0-9][0-9][0-9][;]?", "?", text) # Remove ligature marks '{}': text = re.sub(r"[{]([^{}]*)[}]", r"\1", text) # Map to lowercase: text = text.lower() # Map hooked puffs to normal ones: text = re.sub(r"w", "p", text) text = re.sub(r"z", "f", text) # Map rare uncertain characters to '?': text = re.sub(r"[bj]", "?", text) # Check for parag markers: head = ( re.search(r"^[<][%][>]", text) != None ) tail = ( re.search(r"[<][$][>]$", text) != None ) # Remove all parag markers for now: text = re.sub(r"[<][%$][>]", "", text) # Remove alignment markers: text = re.sub(r"^[«=»]", "", text) text = re.sub(r"[«=»]$", "", text) # Check for invalid characters in input: m_bad = re.search(r"[^-.,ac-ik-z?]", text) if m_bad is not None: ibeg = m_bad.start() data_error(f"invalid character '{text[ibeg]}' at {ibeg}") # Check for irregular spaces in input: m_bad = re.search(r"^[-,.]|[-,.][-,.]|[-,.]$", text) if m_bad is not None: ibeg = m_bad.start(); iend = m_bad.end() data_error(f"improper punctuation '{text[ibeg:iend]}' at {ibeg}") # Replace punctuation: text = re.sub(r"-", ".", text) if utype == "ec": text = re.sub(r"[,.]", "", text) elif utype == "wc": text = re.sub(r"[,]", ".", text) elif utype == "wp": text = re.sub(r"[,]", "", text) else: assert False # Should not happen. else: arg_error(f"invalid {utype = !r}") check_starps_cleanup(text, utype, data_error) return text, head, tail # ---------------------------------------------------------------------- def clean_up_bencao_raw_text(text, utype, data_error): # Cleans up the raw {text} from an SBJ transcription file according to {utype} # for use in Note 077 analysis. Returns the cleaned {text}. # # INPUT TEXT EXPECTED # # For any {utype} the input text may contain editorial annotations and markup. # The details depend on {utype}: # # "ch": the input raw {text} must consist of simplified hanzi # (Chinese characters) in Unicode, with ideographic punctuation # and markup characters [］［；：、，。]. It is assumed that # editorial annotations delimited by ideographic parens（...）were # moved to #-comments, but some tags like （女子）remain, and they # are not removed. The raw {text} should not have embedded ASCII # blanks or other ASCII chars. Example: # # "冬葵子：［味］甘寒。［主］五藏六腑寒热羸瘦。［久服］坚骨长肌肉。（草头）" # # CLEANUP # # Cleanup for both {utype}s implies: # # Replacing all punctuation by spaces. # # CLEANED TEXT # # After this cleanup, if {utype} is "ch", the raw text must # consist entirely of simplified hanzi. Example: # # "冬葵子味甘寒主五藏六腑寒热羸瘦久服坚骨长肌肉" # if utype == "ch": # # Remove annotations (enclosed in ideographic parens): # text = re.sub(r"[（][^（）]*[）]", "", text) # Remove all punctuation: */ text = re.sub(r"[］［；：、，。（）]", "", text) else: arg_error(f"invalid text unit type {utype = !r}") check_bencao_cleanup(text, utype, data_error) return text # ---------------------------------------------------------------------- def normalize_starps_text(text, utype, data_error): # Normalizes the raw {text} from an SPS transcription file according to {utype} # for use in Note 077 analysis. Returns the normalized text # # For all {utype}s, the normalization entails fixing assumed scribal # mistakes and assumed non-significant handwriting variation, # by mapping # # 'g','m' to 'iin' # 'ir' to 'iin' # 'u' to 'n' # 'hh' to 'he' # 'ih' to 'ch' # 'iGh' to 'cGh' for any gallows G. # if utype == "ec" or utype == "wc" or utype == "wp": # Reduce presumed handwriting variants: text = re.sub(r"g", "m", text) text = re.sub(r"u", "n", text) # Correct presumed scribal and transcription errors and abbreviations: text = re.sub(r"ir", "iin", text) text = re.sub(r"m", "iin", text) text = re.sub(r"hh", "he", text) text = re.sub(r"ih", "ch", text) text = re.sub(r"i([ktpf])h", r"c\1h", text) else: arg_error(f"invalid {utype = !r}") return text # ---------------------------------------------------------------------- def normalize_bencao_text(text, utype, data_error): # Normalizes the raw {text} from an SBJ transcription file according to {utype} # for use in Note 077 analysis. Returns the normalized {text}. # # The input text {text} must have the punctuation, including ideographic # brackets '' around keywords 味, 主, 久服, etc. # # Cleanup for both {utype}s implies: # # Deleting entry fields that apparently are not transcribed in the SPS: # # ［味］... [wèi] "taste and warmth" # ［一名］... [yī míng] "another name" # ［生 ］... [shēng] "place of origin" # # Deleting the sub-entry "鸡白蠹：肥猪。" = "jī bái dù: féi zhū." # which is a vet/farming use. # # Normalizing some keys: # # ［治］ [zhì] to ［主］ [zhǔ] # ［主］ [zhǔ] to ［主］ [zhǔ] # ［久服］之 [jiǔ fú] zhī to ［久服］ [jiǔ fú] # ［久食］ [jiǔ shí] to ［久服］ [jiǔ fú] # # if utype == "ch": check_bencao_cleanup(text, utype, data_error) # Remove fields apparently omitted from the SPS: text = re.sub(r"\［(味|一名|生)\］[^］［]*", "", text) # Regularize field keywords: text = re.sub(r"\［主治\］", "［主］", text) text = re.sub(r"\［久食\］", "［久服］", text) # Specific fixes: text = re.sub(r"\［治\］", "［主］", text) # . text = re.sub(r"鸡白蠹[：、，]?肥猪[。]?", "", text) # ROOS text = re.sub(r"（东门上者尤良。）", "", text) # ROOS text = re.sub(r"\［久服之\］", "［久服］", text) # . else: arg_error(f"invalid text unit type {utype = !r}") return text # ---------------------------------------------------------------------- def check_starps_cleanup(text, utype, data_error): # Checks whether {text} is a valid "clean" SPS text according # to the given {utype}. Calls {data-error} if not. # Check for invalid characters: if utype == "ec": m_bad = re.search(r"[^ac-ik-vxy?]", text) elif utype == "wc" or utype == "wp": m_bad = re.search(r"[^.ac-ik-vxy?]", text) else: arg_error(f"invalid {utype = !r}") if m_bad is not None: ibeg = m_bad.start() data_error(f"invalid text char '{text[ibeg]}' at pos {ibeg}") if utype == "wc" or utype == "wp": # Check for improper spacing: pat_bad_punc = r"^[.]|[.][.]|[.]$" m_bad = re.search(pat_bad_punc, text) if m_bad != None: ibeg = m_bad.start() data_error(f"improper punctuation '{text[ibeg]}' at pos {ibeg}") return # ---------------------------------------------------------------------- def check_bencao_cleanup(text, utype, data_error): # Checks whether {text} is a valid "clean" SBJ text according # to the given {utype}. Calls {data-error} if not. # # !!! Currently does not do a thorough check. Impreove! !!! if utype == "ch": # Check for ASCII characters: m_bad = re.search(r"[\001-\377]", text) if m_bad is not None: ibeg = m_bad.start() data_error(f"invalid text char '{text[ibeg]}' at {ibeg}") return # ---------------------------------------------------------------------- def get_parsing_patterns(utype): # Returns # # {pat_line} RE pattern matching a data line, where group 1 # is the locator (minus '<>') and group 2 is the text. # # {pat_unit} RE pattern matching one unit in the raw text. # # {pat_sepa} RE pattern matching any char that serves # as unit separator in the clean text. # # {clean_sepa} String that should separate units in clean text. # # The pattern {pat_sepa} may be {None} # if there are supposed to be no chars in that role. # # See {clean_up_starps_raw_text} and {clean_up_raw_bencao_text} for # the semantics of {utype}. # # The locus ID format {loc} depends on {utype}: # # "ch": the {loc} must be "b{sec}.{subsec}.{lseq}". # # "ec", "wp", or "wc": the {loc} must be "f[0-9]*[rv][0-9]*". # # Unit of text size/position: # # "ch": The units of counting is one hanzi. # # "ec": a single eva character [?a-z]. # # "wc" and "wc": one EVA token, a string [?a-z]+ delimited by '.' # if utype == "ch": pat_loc = r"<(b[1-3][.][0-9][.][0-9][0-9][0-9])>" pat_unit = r"." # Each char is a unit. pat_sepa = None # There are no unit separators on input. clean_sepa = "" # No separators in cleaned text. elif utype == "ec" or utype == "wc" or utype == "wp": pat_loc = r"<(f[0-9]+[rv][0-9]*[.][0-9]+)>" if utype == "ec": pat_unit = r"[a-z?]" # Unit is a single EVA char. pat_sepa = None # There are no unit separators on input. clean_sepa = '' # No unit separator in cleaned text. elif utype == "wc" or utype == "wp": pat_unit = r"[a-z?]+" # Unit is one or more EVA chars. pat_sepa = r"[.]" # In clean text only '.' is separator. clean_sepa = '.' # Unit separator is '.' in cleaned text. else: arg_error(f"invalid {utype = !r}") pat_line = f"{pat_loc}[ ]+(.*)" return pat_line, pat_unit, pat_sepa, clean_sepa # ---------------------------------------------------------------------- def hanzi_per_unit(utype, trimmed): # Nominal average number of units of type {utype} for a Chinese ideogram word. # If {trimmed} assumes that all SBJ fields that are omitted # in the SPS have been deleted fro the SBJ text. if utype == "ch": return 1.0000 elif utype == "wc": return 0.9425 if trimmed else 0.7952 elif utype == "wp": return 1.0660 if trimmed else 0.8994 elif utype == "ec": return 0.1975 if trimmed else 0.1568 else: assert False, f"invalid {utype = !r}" # ---------------------------------------------------------------------- def split_text_into_units(text, utype, pat_unit, pat_sepa, data_error): # Splits cleaned {text} into units of the type specified by {utype}. # Assumes that the txt was cleaned according to {utype}. # # Returns a list {units} with the units, and a count {ct_sepa} of # chars in {text} that were matched by {pat_sepa} (and thus not counted as units). # # Currenly the {utype} must be "ch" (Chinese characters in Unicode) # , "ec", "wc", or "wp" (EVA-encoded Voynichese). # # Loop on units and separators: ct_sepa = 0 units = [] while len(text) > 0: if pat_sepa is not None: m = re.match(pat_sepa, text) if m is not None: assert m.start() == 0 n = m.end() assert n > 0, "pattern {pat_sepa} matched empty" ct_sepa += n text = text[n:] continue m = re.match(pat_unit, text) if m is not None: assert m.start() == 0 n = m.end() assert n > 0, "pattern {pat_unit} matched empty" units.append(text[0:n]) text = text[n:] continue data_error(f"invalid chars = '{text}'") return units, ct_sepa # ---------------------------------------------------------------------- def list_occurrences_in_units(word, units, clean_sepa, utype, data_error): # Returns a list of the occurrences of {word} in # the text that consists of the list {units} of units of type {utype} # preceded, separated, and terminated by {clean_sepa}. # # Note that the bracketing {clean_sepa} make the text not valid. # They are added to simplify matching whole words. # # The {word} may be an RE pattern and/or may include {clean_sepa} # to control matches. # # !!! Misses overlapping occurrences. Is that OK? !!! if len(units) == 0: return [] btext = clean_sepa + clean_sepa.join(units) + clean_sepa if utype == "ch": oclist = list_ch_occurrences(word, btext) elif utype == "ec": oclist = list_ec_occurrences(word, btext) elif utype == "wc" or utype == "wp": oclist = list_wc_or_wp_occurrences(word, btext) else: arg_error(f"invalid {utype = !r}") return oclist # ---------------------------------------------------------------------- def list_ch_occurrences(word, btext): # Finds occurrences of {word} as string in the Unicode CHU-8 hanzi # {btext}, returning their positions. The text {btext} should have no # iso-latin-1 chars or hanzi punctuation. # # Positions are measured in hanzi chars. oclist = list(( p.start() for p in re.finditer(word, btext) )) return oclist # ---------------------------------------------------------------------- def list_ec_occurrences(word, btext): # Finds occurrences of {word} as string in the EVA {btext}, returning # their positions. Assumes that all blanks and EVA puntuation [-,.] # have been deleted. # # Positions are measured in EVA character counts oclist = list(( p.start() for p in re.finditer(word, btext) )) return oclist # ---------------------------------------------------------------------- def list_wc_or_wp_occurrences(word, btext): # Finds occurrences of {word} in the EVA {btext}, returning their # positions. # # Assumes that blanks, other EVA junk, and unwanted separators ([,] # for "wp") have been deleted, relevant unit separators ([-,.] for # "wc", [-.] for "wp") have been mapped to '.', and there is a single # '.' before, between, and after every unit, # # Positions are measured in EVA words. Unmatched EVA word suffixes or # prefizes are counted as 0.5 word. assert btext[0] == '.' and btext[-1] == '.', "cleanup failed" debug = False # We must loop on words: rest = btext oclist = [] pref = "" while True: if debug: err.write(f"!* '{pref}|{rest}'\n") m = re.search(word, rest) if m is None: break ix = m.start(); fx = m.end() if debug: err.write(f"!* {ix = } {fx = }\n") assert ix < len(rest) # If {word} matched just before '.', adjust {ix} to exclude the '.': if rest[ix] == '.': ix = ix + 1 # If {word} matched just after '.', adjust {fx} to exclude the '.': if fx < len(rest) and rest[fx-1] == ' ': fx = fx - 1 if debug: err.write(f"!* adjusted {ix = } {fx = }\n") # Count whole words before occurrence of {word}:] pref = pref + rest[0:ix]; core = rest[ix:fx]; rest = rest[fx:] if debug: err.write(f"!* '{pref}[{core}]{rest}'\n") oc = len(re.findall(r"[.]", pref)) - 1 if debug: err.write(f"!* {oc = }'\n") # If {word} matched only a suffix, count the prefix as half a word: if pref != '' and pref[-1] != '.': oc += 0.5 oclist.append(oc) pref = pref + core return oclist # ---------------------------------------------------------------------- def compute_string_sizes(strings): # Given a list {strings} of strings, returns a list of the lengths of # those strings, and their total. sizes = [ len(st) for st in strings ] tsize = 0; for sz in sizes: tsize += sz return sizes, tsize # ---------------------------------------------------------------------- def round_est_size(sz, eps_log): # Rounds an estimated size after adding # {eps_log} in log scale. Rounds down if {eps_log} # is negative, up otherwise. tiny = 1.0e-100 if eps_log < 0: rsz = int(floor(exp(log(sz+tiny) + eps_log))) assert rsz <= sz elif eps_log > 0: rsz = int(ceil(exp(log(sz+tiny) + eps_log))) assert rsz >= sz else: rsz = int(sz + 0.5) return rsz # ---------------------------------------------------------------------- def scale_size_fuzzy(size, scale, eps): # Scales the given {size} by the given {scale} with some # slack up or down. Specifically {±eps} times an estimate of the # uncertainty in log scale. # # The {size} may be an integer or a range represented as a # pair of integers {(lo,hi)}. The result will always be a range # Accept integer as singleton ranges. if isinstance(size, int): size = (size,size,) loz = size[0]; lo_dev_log = compute_dev_of_log(loz) los = round_est_size(scale * loz, -eps * lo_dev_log) hiz = size[1] hi_dev_log = compute_dev_of_log(hiz) his = round_est_size(scale * hiz, +eps * hi_dev_log) return (los,his,) # ---------------------------------------------------------------------- def scale_sizes_fuzzy(sizes, scale, eps): # Given a list {sizes} of integers or integer ranges, # scales each element {sz} with {scale_size_fuzzy(sz,scale,eps} # (quod videt). new_sizes = [] for sz in sizes: new_sizes.append(scale_size_fuzzy(sz, scale, eps)) return new_sizes # ---------------------------------------------------------------------- def compute_dev_of_log(es): # Assumes that {es} is the expected value of a size variable with a # log-normal distribution with deviation proportional to {sqrt(es)}. # Returns the approximate deviation of the distributon of the log of # the size. debug = False if debug: err.write(f"!@ {es = :3d}") eps = 1 es0 = hypot(es, eps) # In case {es} is zero. es0_log = log(es0) if debug: err.write(f" {es0 = :5.1f} {es0_log = :+7.3f}") var_abs = 0.10 * es0 # Rough guess - to be improved. dev_abs = sqrt(var_abs) if debug: err.write(f" {dev_abs = :7.3f}") es1 = hypot(es0 + dev_abs, eps) es1_log = log(es1) if debug: err.write(f" {es1 = :5.1f} {es1_log = :+7.3f}") dev_log = es1_log - es0_log if debug: err.write(f" {dev_log = :+7.3f}\n") return dev_log # ---------------------------------------------------------------------- def compute_frac_error(s, es): # Fractional error between a size {s} and the expected size range {es}, # relative to the estimated deviation of the latter, in log scale. debug = False if debug: err.write(f"!~ {s = :3d} es = {es[0]:3d}..{es[1]:3d}") assert len(es) == 2 and es[0] <= es[1] avg = es[0] if s < es[0] else es[1] if s > es[1] else s if debug: err.write(f" {avg = :3d}") if s == avg: frac = 0 else: eps = 1 s0 = hypot(s, eps) s0_log = log(s0) if debug: err.write(f" {s0 = :5.1f} {s0_log = :+7.3f}") avg0 = hypot(avg, eps) avg0_log = log(avg0) if debug: err.write(f" {avg0 = :5.1f} {avg0_log = :+7.3f}") dev_log = compute_dev_of_log(avg) if debug: err.write(f" {dev_log = :7.3f}") frac = (s0_log - avg0_log)/dev_log if debug: err.write(f" {frac = :7.3f}\n") return frac # ---------------------------------------------------------------------- def compute_rorre_carf(frac, es): # Inverse of fractional error eps = 1 if frac < 0: dev_log = compute_dev_of_log(es[0]) y = frac*dev_log + log(hypot(es[0], eps)) z = exp(y) s = 0 if z <= eps else sqrt(z*z - eps*eps) elif frac > 0: dev_log = compute_dev_of_log(es[1]) y = frac*dev_log + log(hypot(es[1], eps)) z = exp(y) s = 0 if z <= eps else sqrt(z*z - eps*eps) else: # Geometric mean of the range end values: s = sqrt(hypot(es[0], eps)*hypot(es[1], eps)) assert s >= eps s = sqrt(s*s - eps*eps) return s # ---------------------------------------------------------------------- def parse_size_ranges(szranges_str): # The list {szranges_str} must be a string consisting of {neg} items # separated by commas, where each item is either an integer or a pair of # integers separated by '..'. They are converted to a list # {szranges[0..neg-1]} of integer pairs {(lo,hi)}. # xitems = re.split(r"[,]", szranges_str) szranges = [] for xit in xitems: xit = xit.strip() if xit != "": m = re.fullmatch(r"([0-9]+)[.][.]([0-9]+)", xit) if m != None: it_min = int(m.group(1)) it_max = int(m.group(2)) assert 1 <= it_min and it_min <= it_max, f"bad expected range {it_min}..{it_max}" szranges.append((it_min,it_max,)) else: it_val = int(xit) szranges.append((it_val,it_val,)) return szranges # ---------------------------------------------------------------------- def format_range(szrange): los = szrange[0]; his = szrange[1] if los == his: return str(los) else: return str(los) + ".." + str(his) # ---------------------------------------------------------------------- def format_size_ranges(szranges): szranges_str = '' for szrange in szranges: if szranges_str != '': szranges_str += ',' egs_str = format_range(szrange) szranges_str += egs_str return szranges_str # ---------------------------------------------------------------------- def test_stuff(): def data_error(msg): err.write(f"** {msg}\n"); assert False # .................................................................... err.write("----------------------------------------\n") err.write("testing compute_frac_error, compute_rorre_carf ...\n") for gs in range(30): fr = compute_frac_error(gs, (15,20,)) rf = compute_rorre_carf(fr, (15,20,)) err.write(f"{gs = :4d} {fr = :7.3f} {rf = :7.3f}\n") err.write("----------------------------------------\n") err.write("testing compute_dev_of_log ...\n") for k in range(5): esk = 5*k esk_log = log(hypot(esk, 1)) dev_log = compute_dev_of_log(esk) err.write(f"{esk = :4d} {esk_log = :7.3f} {dev_log = :7.3f}\n") err.write("----------------------------------------\n") err.write("testing get_parsing_patterns, split_text_into_units ...\n") for utype in ( "wc", "wp", "ec", "ch", ): err.write(f"!* counting '{utype}' units\n") pat_line, pat_unit, pat_sepa, clean_sepa = \ get_parsing_patterns(utype) if utype == "ec" or utype == "wc" or utype == "wp": text = "foo,par.paz-q.?ofoo,parrifoo" words = [ "foo", "par", ] if utype == "wc" or utype == "wp": words += [ "[.]foo", "foo[.]", "[.]foo[.]", "foo[.]par", ] assert clean_sepa == '.' elif utype == "ec": assert clean_sepa == '' elif utype == "ch": text = "白石英：［味］甘微温。［主］消渴，阴痿不足，欬逆白石，青石黑石脂等：［味］甘平。［主］黄疸，泄利" words = [ "白石", "主" ] assert clean_sepa == '' else: assert False err.write(f"!* raw text = '{text}')\n") text = clean_up_raw_text(text, utype, data_error) err.write(f"!* cleaned text = '{text}')\n") units, ct_sepa = \ split_text_into_units(text, utype, pat_unit, pat_sepa, data_error) err.write(f"!* {ct_sepa = } {units = !r}\n") err.write("\n") for word in words: err.write(f"!* looking up '{word}'\n") oclist = list_occurrences_in_units(word, units, clean_sepa, utype, data_error) err.write(f"!* found = {oclist}\n") err.write("\n") return # ---------------------------------------------------------------------- if len(sys.argv) > 1 and sys.argv[1] == "SPF.TEST": test_stuff()