#!/usr/bin/env python

'''
kuceradat-0668.txt retrieved from
https://ota.bodleian.ox.ac.uk/repository/xmlui/handle/20.500.12024/0668?show=full

Description of format:
https://ota.bodleian.ox.ac.uk/repository/xmlui/bitstream/handle/20.500.12024/0668/kuceradoc-0668.txt?sequence=5&isAllowed=y
'''

import os

def kfSubsets(fname='kuceradat-0668.txt'):
    kfList = []

    if not os.path.exists(fname):
        print('%s: no such file .' % fname)
        return

    wTot = 0
    with open(fname, 'r', encoding='utf-8', errors='ignore') as f:
        for line in f:
            parts = line.strip().split() # Clean up white space padding
            # Skip noise rows, lines containing arrows/markers, or short lines
            if len(parts) < 4 or '^' in line or '`' in line:
                continue
            wTot += 1
            try:
                freq = int(parts[0]) # Frequency count in first column.
            except ValueError:
                continue # Skip non-numbered (non-data) lines.
            if freq < 10: # Ignore little used words.
                continue

            word = parts[3] # Word is at index 3.
            if len(word) < 4: # Ignore words under 4 characters long.
                continue

            # 1988 Filter: Clean out mathematical notations, symbols, and
            # code tags.  Keep pure alphabetic words.
            wd = word.replace('-', '').replace("'", '')
            if not wd.isalpha():
                continue

            # Keep both original word case and processed relative frequency
            kfList.append((word, freq))

    print('Full K-F database: %d words.' % wTot)
    print('Trimmed K-F database: %d words.' % len(kfList))

    # Generate 3 subsets as in my thesis.  Every 10th word starting at...
    subset1 = kfList[0::10]  # ... 1st word.
    subset2 = kfList[3::10]  # ... 4th word.
    subset3 = kfList[6::10]  # ... 7th word.

    # Save datasets
    datafiles = {
        'subset1.txt': subset1,
        'subset2.txt': subset2,
        'subset3.txt': subset3
    }

    full = 'kffull.txt'
    with open(full, 'w') as out:
        f = 0
        for w, freq in kfList:
            print('%s %d' % (w, freq), file=out)
            f += freq
        print('Exported %d rows to %s' % (wTot, full))
        print('Total freq: %d.' % f)

    for fname, data in datafiles.items():
        f = 0
        with open(fname, 'w', encoding='utf-8') as out:
            for w, freq in data:
                print('%s %d' % (w, freq), file=out)
                f += freq
        print('Exported %d rows to %s' % (len(data), fname))
        print('Total freq: %d.' % f)

if __name__ == '__main__':
    kfSubsets('kuceradat-0668.txt')