#!/usr/bin/env python

import numpy as np
import os, sys

CM_NO = 16
TOK_NO = 10

class State:

    def __init__(self, num, stActions):
        self.num = num
        self.stActions = stActions
        self.transitions = {} # List of transition:state pairs.

    def __str__(self):
        s = 'state %d: actions %s, transitions:\n' \
            % (self.num, hex(self.stActions))
        for k in self.transitions.keys():
            s += ' %02x -> state %d\n' % (k, self.transitions[k])
        return s

    def actions(self):
        return self.stActions

    def addTransitions(self, trigDict):
        self.transitions = trigDict

    def nextState(self, trigger):
        return self.transitions[trigger]

class Fsm:

    def __init__(self):
        self.fsm = self.mkFsm()
        self.st = self.fsm[0];

    def __str__(self):
        s = ''
        for st in self.fsm:
            s += str(st) + '\n'
        return s

    def mkFsm(self):
        fsm = []

        # Create the seven states and actions upon arrival.
        # Action bits are: ref ef i1 i2 ie ab
        # (0x20) ref: set ref
        # (0x10) ef: set ef
        # (0x08) i1: increment token pointer by 1.
        # (0x04) i2: increment token pointer by 2.
        # (0x02) ie: increment error counter by 1.
        # (0x01) ab: abort search.
        fsm.append(State(0, 0x20)) # Set ref.
        fsm.append(State(1, 0x01)) # Set abort.
        fsm.append(State(2, 0x00))
        fsm.append(State(3, 0x16)) # Set ef, i2, ie.
        fsm.append(State(4, 0x06)) # Set i2, ie.
        fsm.append(State(5, 0x08)) # Set i1.
        fsm.append(State(6, 0x0a)) # Set ie, i1.

        # Trigger/destinations for state.
        # Trigger bits are: i i+1 i-2 ref ef
        # (0x10) i:   expected char (token pointer).
        # (0x08) i+1: next expected char.
        # (0x02) i-2: second previous expected char.
        # (0x02) ref: ref setting.
        # (0x01) ef:  ef setting.
        t = {0x00:0, 0x04:0, 0x08:3, 0x0c:3, 0x1c:5}
        fsm[0].addTransitions({0x02:1, 0x06:1, 0x0a:4, 0x0e:4, 0x1e:6})
        fsm[1].addTransitions(t)
        fsm[2].addTransitions(t)
        fsm[3].addTransitions({0x01:0, 0x05:2, 0x0d:2, 0x09:4, 0x1d:5})
        fsm[4].addTransitions(t)
        fsm[5].addTransitions(t)
        fsm[6].addTransitions(t)

        return fsm

    def transit(self, state, trigger):
        if state is None:
            state = self.fsm[2] # efc.c activates as st->next->next.
        trigger = 0x1c|trigger if 0x10&trigger else trigger # Mask don't-cares.
        stNum = state.nextState(trigger)
        return self.fsm[stNum]

class TokenFollower:

    def __init__(self, iCell):
        self.reset()
        self.iCell = iCell # Index number of cell.
        self.aNominal = 0 # Activation order of this token follower.

    def abort(self, a=None):
        if a is not None:
            self.isAbort = a
        return self.isAbort

    def active(self, act=None):
        if act is not None:
            self.isActive = act
        return self.isActive

    def cell(self):
        return self.iCell

    def errFlags(self, ef=None, ref=None):
        if None not in [ef, ref]:
            self.ef = ef
            self.ref = ref
        return self.ef, self.ref

    def errCnt(self):
        return self.errs

    def follow(self, i, ip1, im2):
        '''Token follows FSM and moves forward one state and perform actions
        upon entry to new state.

        Inputs:
          i (boolean): indicates if char matcher i does/doesn't match.
          ip1 (boolean): indicates if char matcher i+1 does/doesn't match.
          im2 (boolean): indicates if char matcher i-2 does/doesn't match.
        '''

        global fsm # One FSM used by everyone.

        trigger = (i<<4) | (ip1<<3) | (im2<<2) | (self.ref<<1) | self.ef
        self.state = fsm.transit(self.state, trigger) # Move one state in FSM.
        actions = self.state.actions()

        # Act on new state's actions.
        self.ref     = (0x20 & actions) == 0x20 # REF, otheR Error Flag.
        self.ef      = (0x10 & actions) == 0x10 # EF, Error Flag.
        self.isAbort = (0x01 & actions) == 0x01 # Abort search.
        if 0x08 & actions:
            self.iCell += 1
        if 0x04 & actions:
            self.iCell += 2
        if 0x02 & actions:
            self.errs += 1

    def order(self, aNominal=None):
        if aNominal is not None:
            self.aNominal = aNominal
        return self.aNominal

    def reset(self):
        self.isActive = False # Initially in active.
        self.state = None     # Current state EFC FSM.
        self.errs = 0         # Total number of ref and ef occurrences.
        self.iCell = 0        # Cell that token pointer is pointing to.
        self.ref = False      # Deletion/transposition has/hasn't occurred.
        self.ef = False       # Insertion/substituion has/hasn't occurred.
        self.isAbort = False  # Do/don't abort search.

class CharacterMatcher:
    '''Currently just performs simple character match, but can be extended
    to match classes of characters like alpha, numeric, etc., as described
    in Mules' thesis.
    '''

    def __init__(self, chList):
        self.chList = chList # Characters that can be matched.

    def match(self, c):
        return c in self.chList

class StringMatcher:

    def __init__(self, word):
        self.enabled = False

        # Instantiate array of token followers.
        self.tf = []
        for cell in range(TOK_NO):
            self.tf.append(TokenFollower(cell))

        # Instantiate array of character matchers.
        self.cm = []
        for i in range(len(word)):
            self.cm.append(CharacterMatcher(word[i]))

    def enable(self, en):
        self.enabled = en

    def feed(self, c, aOrd):
        '''Feed substring matcher a new incoming character.
        '''

        # Activate a new token follower.
        aNom = -1 # Slot index activated this call, or -1 if pool was full.
        for t in range(TOK_NO):
            if not self.tf[t].active():
                tok = self.tf[t]
                tok.reset()
                tok.active(True)
                tok.order(aOrd)
                aOrd += 1 # Order of activation, can be > TOK_NO.
                aNom = t # Tok follower index of activation.
                break
        else:
            print('Ran out of token followers.')

        # Count active token followers for stats gathering.
        nActive = 0
        for t in range(TOK_NO):
            if self.tf[t].active():
                nActive += 1

        # Move each token follower 1 state through FSM.
        match = False
        for iTok in range(TOK_NO):
            tok = self.tf[iTok]
            if self.tf[iTok].active():
                cm = self.cm # Shorter name.
                cur = tok.cell() # Cell pointed to by token follower.
                i = ip1 = im2 = False
                if cur < len(cm) and cm[cur].match(c):
                    i = True
                if cur < len(cm)-1 and cm[cur+1].match(c):
                    ip1 = True
                if cur > 1 and cm[cur-2].match(c):
                    im2 = True
                tok.follow(i, ip1, im2) # Move 1 state in FSM.

                # Check if match occurred.
                nErr = 0
                if tok.errCnt() == 4:
                    tok.abort(True)
                elif tok.cell() >= len(cm): # Candidate word matched target.
                    match = True
                    nErr = tok.errCnt()
                    break

            if tok.abort(): 
                tok.reset()
                continue
        # iTok is index of token follower that matched/aborted.
        return match, nActive, nErr, iTok, aOrd, aNom

    def match(self, c, t):
        val = 0
        cm = self.cm

        ef, ref = self.errFlags()
        i = self.tf[t].cell()

        if cm[i].match(c): # c matches char matcher i.
            val |= 0x10
        if i < len(cm)-1 and cm[i+1].match(c): # c matches char matcher i+1.
            val |= 0x08
        if i > 1 and cm[i-2].match(c): # c matches char matcher i-2.
            val |= 0x04
        if ref:
            val |= 0x02
        if ef:
            val |= 0x01
        return val

    def reset(self):
        for i in range(TOK_NO):
            self.tf[i].reset()

class Stats:
    '''This class is instantiated for target words of a particular length.
    '''

    def __init__(self):
        '''As characters stream in, token followers are activated and
        deactivated in an unpredictable way
        '''
        # Words mismatch with cardinal (actual) token follower i.
        self.nNom = np.zeros(TOK_NO) # Number of mismatches.
        self.pNom = np.zeros(TOK_NO) # Probability of mismatches.
        # Words that matched with ordinal token i.
        self.nNom = np.zeros(TOK_NO) # Number of mismatches.
        self.pNom = np.zeros(TOK_NO) # Probability of mismatches.
        # Mismatching words with 0-3 errors.
        self.nErrors = np.zeros(4) # Count of errors.
        self.pErrors = np.zeros(4) # Probability of errors.

        self.mismatches = 0 # Mismatches for this word length.
        self.mostUsed = 0 # Most tokens used in any match.
        self.nToksUsed = np.zeros(TOK_NO) # Tok followers used for match.
        self.wds = 1e6 # Approx number of words in Kucera-Francis db.
        self.wds = 1e5 # Probably what I used in the thesis!
        self.wds = 64795 # Probably what I used in the thesis!

    def nominal(self, iTok, occTarg=None, occCand=None):
        '''This token is a cardinal match.  That is, regardless of what
        place in the order of activation, it is the one that performed the
        final target/cadidate (mis-)match.
        '''

        if occTarg is not None:
            self.nNom[iTok] += 1
            self.pNom[iTok] += (occTarg*occCand)/self.wds**2
        return self.nNom[iTok], self.pNom[iTok]

    def errors(self, nErr, occTarg=None, occCand=None):
        if occTarg is not None:
            self.nErrors[nErr] += 1
            self.pErrors[nErr] += (occTarg*occCand)/self.wds**2
        return self.nErrors[nErr], self.pErrors[nErr]

    def misMatches(self, occTarg=None, occCand=None):
        if occTarg is not None:
            self.mismatches += (occTarg*occCand)/self.wds**2
        return self.mismatches

    def ordinal(self, iTok, occTarg=None, occCand=None):
        '''This subroutine deals with nominal, as in named, matches.  That
        is, regardless of what its actual hardware order, it is the name of
        the one that performed the final target/cadidate (mis-)match in some
        sequence of token follower activations.

        Think of the hardware where each token follower in turn is named, 0,
        1, 2, ... and this routine returns both the number and probability
        of matches made by this follower.
        '''

        if occTarg is not None:
            self.nOrd[iTok] += 1
            self.pOrd[iTok] += (occTarg*occCand)/self.wds**2
        return self.nOrd[iTok], self.pOrd[iTok]

    def toksMost(self, used=None):
        if used is not None:
            self.mostUsed = max(self.mostUsed, used)
        return self.mostUsed

    def toksUsed(self, j, occTarg=None, occCand=None):
        '''Histogram of peak simultaneous token-follower usage across
        candidate words, weighted by occTarg*occCand. j is the peak
        count minus 1 (0 -> 1 token used, 1 -> 2 tokens used, etc.),
        matching efc.c's wd_info[wd_len].toks_used[tok_high].
        '''
        if occTarg is not None:
            self.nToksUsed[j] += (occTarg*occCand)/self.wds**2
        return self.nToksUsed[j]

###########################################################################

def cli(argv):
    prog = argv[0]
    argv = argv[1:]
    print(prog, argv)
    i = 0
    retro = False
    while i < len(argv):
        arg = argv[i]
        if arg[0] != '-':
            break
        elif arg == '-1988': # Python data type of incoming IQ.
            i += 1
            retro = True
    print(arg)
    fname = arg
    return retro, fname

def efc(kfDb, mike1988):
    global fsm # One FSM used by everyone.

    # Prepare to gather statisitcs, one set per word length.
    stats = []
    for _ in range(16):
        stats.append(Stats())

    fsm = Fsm()
    fTarg = open(kfDb, 'r') # Kucera-Francis as target words.
    fCand = open(kfDb, 'r') # Kucera-Francis as candidate words.

    # Global stats for final summary.
    matchingTok = np.zeros(5*TOK_NO + 1)
    nomTok = np.zeros(TOK_NO + 1)

    while True: # Loop through target words.
        target, occTarg = wdGet(fTarg, mike1988)
        if target is None:
            break # Done!
        sm = StringMatcher(target) # New string matcher.
        tlen = len(target) - 1

        while True: # Loop through candidate words.
            cand, occCand = wdGet(fCand, mike1988) # Next candidate word, freq pair.
            if cand is None:
                break # Done!
            isSelf = (cand == target)
            if isSelf and not mike1988:
                # Avoid my 1988 bug!  Info on matches should never be kept.
                # My old code partially did.  Please don't take my degree
                # away. :-D
                continue

            # Feed word, char by char, to substring matcher.
            most = 0
            tokHigh = -1 # Most followers activated for this word.
            ord0 = 0 # Token follower activation order for this word.
            for ch in cand: # Use each character.
                match, nActive, nErr, iTok, ord1, aNom = sm.feed(ch, ord0)
                ord0 = ord1 # Age counter from feed().
                most = max(most, nActive) # Most token followers active.
                tokHigh = max(tokHigh, aNom)
                if match:
                    break

            stats[tlen].toksMost(most)
            if tokHigh >= 0:
                stats[tlen].toksUsed(tokHigh, occTarg, occCand)
            if match: # target & candidate words match!
                if not isSelf:
                    stats[tlen].misMatches(occTarg, occCand)
                stats[tlen].errors(nErr, occTarg, occCand)
                stats[tlen].nominal(iTok, occTarg, occCand)
                if aNom != -1: # Token follower activated on final char.
                    stats[tlen].nominal(iTok, occTarg, occCand)
                if mike1988 or not isSelf:
                    matchOrder = sm.tf[iTok].order() + 1
                    wds = stats[tlen].wds
                    matchingTok[matchOrder] += (occTarg*occCand)/wds**2
                    nomTok[iTok+1] += occTarg/stats[tlen].wds
            sm.reset()

        fCand.seek(0) # Back to start of K-F db.
    fCand.close()
    fTarg.close()
    results(stats, matchingTok, nomTok)

def results(s, matchingTok=None, nomTok=None):

    # Mismatches per word length.
    toksSubtot = np.zeros(TOK_NO)
    tokErrs = np.zeros(4)
    mmTot = wdTot = toksTot = allErrs = avgTot = 0
    for i in range(3, 16):
        print('\n%d letter target string:\n' % (1+i))
        mm = s[i].misMatches()
        if mm > 0:
            print('  %1.8f mismatches.\n' % mm)
            mmTot += mm

        # Tokens used per word length.
        tmpWdTot = avgWd = 0
        for j in range(TOK_NO):
            pUsed = s[i].toksUsed(j)
            if pUsed == 0: # No words of len i peaked at j+1 tok followers.
                continue
            avgWd  += pUsed*(j+1) # Fraction of matches using j+1 tok followers.
            avgTot += pUsed*(j+1)
            toksSubtot[j] += pUsed
            toksTot       += pUsed
            tmpWdTot      += pUsed
            wdTot         += pUsed

        for j in range(TOK_NO):
            pUsed = s[i].toksUsed(j)
            if pUsed == 0:
                continue
            print('  %1.8f used %2d tokens.' % (pUsed/tmpWdTot, j+1))

        tmpTotToks = 0
        for j in range(4):
            nErr, pErr = s[i].errors(j)
            if nErr == 0:
                continue
            tmpTotToks += pErr
            tokErrs[j] += pErr
            allErrs    += pErr

        print()
        for j in range(4):
            nErr, pErr = s[i].errors(j)
            if nErr == 0:
                continue
            print('  %1.8f of matching tokens had %d errors.'
                % (pErr/tmpTotToks, j))
        if (avgWd > 0):
            print('\n Average: %1.8f tokens per %d-letter word.'
                % (avgWd/tmpWdTot, i+1))

    # Overall summary across all word lengths.
    print('\n%1.8f total mismatches.\n' % mmTot)
    for j in range(TOK_NO):
        if toksTot > 0:
            print('%1.8f used %d tokens.' % (toksSubtot[j]/toksTot, j+1))
    if wdTot > 0:
        print('\nAverage: %1.8f tokens per word.' % (avgTot/wdTot))
    mstTok = 0
    for j in range(TOK_NO):
        if toksSubtot[j] > 0:
            mstTok = j
    print('Most:    %d tokens.\n' % (mstTok+1))

    for j in range(4):
        if allErrs > 0:
            print('%1.8f had %d errors.' % (tokErrs[j]/allErrs, j))

    if matchingTok is not None and nomTok is not None:
        matchTokTot = matchingTok.sum()
        nomTokTot = nomTok.sum()
        print()
        for j in range(1, len(matchingTok)):
            if matchingTok[j] and matchTokTot > 0:
                print('%1.8f words were matched with activated token %d'
                    % (matchingTok[j]/matchTokTot, j))
        print()
        for j in range(1, TOK_NO+1):
            if nomTok[j] and nomTokTot > 0:
                print('%1.8f words were matched with the real token %d'
                    % (nomTok[j]/nomTokTot, j))

def wdGet(f, mike1988):
    line = f.readline()
    if line == '':
        return None, None
    word, freq = line.split()
    if mike1988:
        word = word[:CM_NO-1] # Bug in my 1988 thesis. :-o
    else:
        word = word[:CM_NO]
    freq = int(freq)
    return word, freq

def main(argv):
    mike1988, kfDb = cli(argv)
    efc(kfDb, mike1988)

if __name__ == '__main__':
    main(sys.argv)