basic_str_preprocess.py

"""
Contains classes
    - BasicTextCleaner: removes digits, punctuation, stopwords in a string
    - BasicTokenizer: creates one hot encoding of sentences
    - BagOfWordsTokenizer: creates bag of words representation (1-grams) of string
    - BagofNGrams: tokenizer to make bag of ngrams embedding represntation of a string
"""

import numpy as np
import nltk
import functools
import string
import re

class BasicTextCleaner:
    """
    This is text preprocesser that can perform the following tasks:
        - remove digits
        - remove puncutation
        - remove stopwords
    Inputs:
         - text -> str - this is the text to be processed
         - lower -> bool - indicate if the string should be made lower or not
                           default is True

    Attrs:
        - self.text
        - self.lower

    Methods:
        - process_text - returns string of fully processed text (str)
        - remove_digits - returns only digit removal (str)
        - remove_punctuation - returns only punctuation removal (str)
        - remove_stopwords - removes only stopword removal (str)

    Basic use example:
    text = 'A 5 inch mouse (!) jumped over the LAZY dog'
    Cleaner = BasicTextCleaner(text, lower=True)
    print(Cleaner.process_text())
    > 'a inch mouse jumped over the lazy dog'

    """
    def __init__(self, text, lower=True):
        self.text = text
        self.lower = lower

    def process_text(self):
        #set string to lower case if self.lower is true
        if self.lower:
            self.text = self.text.lower()

        #uses _compose to create a sequence of functions to be
        #applied to the inputs. Additional functions could
        #easily be added to this sequence of functions
        #they will be applied in order (similar to tf.Sequential)
        composed_fn = self._compose(
            self.remove_digits,
            self.remove_punctuation,
            self.remove_stopwords)

        return composed_fn(self.text)

    def _compose(self, *functions):
        #returns a function that sequentially applies each function to an input
        return functools.reduce(lambda f, g: lambda x: g(f(x)), functions)    

    def remove_stopwords(self):
        stopwords_en = set(nltk.corpus.stopwords.words('english'))
        removed = [word for word in self.text.split() if word not in stopwords_en]
        return ' '.join(removed)

    def remove_digits(self):
        return ''.join([c for c in self.text if not c.isdigit()])

    def remove_punctuation(self):
        return re.sub('[' + re.escape(string.punctuation) + r'\\\n]', ' ', self.text)

class BasicTokenizer:
    """
    Creates a vocabulary dictionary from string input and then 
    generates one-hot-encoding for words in sentences. 

    Input
        - text -> str

    Attrs:
        - self.text - input text
        - self.vocab - initially set to None, returns vocab dict after it is generated
    
    Methods
        - tokenize(self) - builds vocab from self.text and then returns one
                           hot representation of self.text as numpy array
        - _build_vocab(self) - generates vocab based on self.text
        - word2num(word) - generates the word index number (ex: the -> 5)
        - num2word(nuM) - generates the word from a word index number (ex: 5 -> the)

    Use example:
    text = 'A 5 inch mouse (!) jumped over the LAZY dog'
    tok = BasicTokenizer(text)
    one_hot_embedding = tok.tokenize()

    """

    def __init__(self, text):
        self.text = text
        self.vocab = None

    def tokenize(self):
        if not self.vocab:
            self._build_vocab()

        #converts list of words to list of numbers (ints)
        nums = [self.word2num(w) for w in self.text.split(' ')]

        n = len(nums)
        #create zero array of shape(num_words, vocab_size)
        encoding = np.zeros((len(self.vocab),n))
        #for each word, make it's index = 1 (for the one hot encoding)
        for i in range(n):
            encoding[nums[i]][i] = 1
        return encoding #shape(num_words, vocab_size)
    
    def _build_vocab(self):
        words = enumerate(set(self.text.split(' ')))
        self.vocab = {word:num for num, word in words}
    
    def word2num(self, word):
        if not self.vocab:
            self._build_vocab()
        #for each word return the vocab dictionary number 
        if word in self.vocab.keys():
            return self.vocab[word]
        return None
    
    def num2word(self, num):
        if not self.vocab:
            self._build_vocab()
        #iterate through items and return the word when index number is found
        #not a very performant implementation.
        for word, number in self.vocab.items():
            if num == number:
                return word
        return None

class BagOfWordsTokenizer(BasicTokenizer):
    """
    Creates bag of words representation of an input text.
    Inputs - text, str

    Attrs:
        self.text
        self.vocab (default None, generated with function calls)

    Methods 
        - tokenize -> returns bag of words (list of ints)

    Inherited Methods
        - _build_vocab(self) - generates vocab based on self.text
        - word2num(word) - generates the word index number (ex: the -> 5)
        - num2word(nuM) - generates the word from a word index number (ex: 5 -> the)

    Use example:
    text = 'the the big cat ran over the lazy dog and then the dog cowered'
    tok = BagOfWordsTokenizer(text)
    print(tok.tokenize()
    > [1. 1. 1. 1. 1. 2. 1. 4. 1. 1.]
    """
    def __init__(self, text):
        super().__init__(text)

    def tokenize(self):
        if not self.vocab:
            self._build_vocab()

        #converts list of words to list of numbers (ints)
        nums = [self.word2num(w) for w in self.text.split(' ')]

        n = len(nums)
        #create zero array of shape(vocab_size)
        encoding = np.zeros((len(self.vocab)))
        #for each num (aka: word) add 1 to it's index in the encoding
        for i in range(n):
            encoding[nums[i]] += 1
        return encoding #shape(vocab_size)    

class BagofNGrams(BasicTokenizer):
    """
    Tokenizer to make ngrams
    Inputs:
        text - str
        n - int how big grams should be
    Attrs:
        self.text - str, input text
        self.vocab - default none, set to dict(tuple:int) by _build_vocab
        self.n - int, how big grams should be
    
    Methods:
        get_ngrams - returns list of tuples of ngrams in self.text
        _build_vocab(self, corpus=None)
            - builds ngram dict (vocab)
            - if corpus = str, will create new vocab dict
        tokenize(self, text=None)
            - returns embedding of self.text
            - if text = str, will initialize new vocab dict

    Inherited methods:
        - word2num(ngram) - generates the ngram index number (ex: the -> 5)
        - num2word(num) - generates the ngram from a ngram index number (ex: 5 -> the)


    Example usage to make 2grams:
    s = 'James is the best person ever the best person ever'
    tok = BagofNGrams(s, 2)
    print(tok.tokenize())
    """
    def __init__(self, text, n):
        super().__init__(text)
        self.n = n

    def get_ngrams(self):
        words = self.text.split()
        return [tuple(words[i:i+self.n]) for i in range(len(words)-self.n)]

    def _build_vocab(self, corpus=None):
        if corpus:
            self.text = corpus

        self.vocab = {ngram:num for num, ngram in enumerate(set(self.get_ngrams()))}

    def tokenize(self, text=None):
        if not self.vocab:
            self._build_vocab(corpus = text)

        nums = [self.word2num(ngram) for ngram in self.get_ngrams()]

        #create zero array of shape(vocab_size)
        encoding = np.zeros((len(self.vocab)))
        #for each num (aka: word) add 1 to it's index in the encoding
        for i in range(len(nums)):
            encoding[nums[i]] += 1
        return encoding #shape(vocab_size)    

class TF_IDF_Tokenizer:
    """
    This is a tokenizer that takes a list of strings (a corpus of documents).
    It generates a vocabulary then returns an embedding vector for a document
    based on this vocab with calculated TF-IDF.

    Inputs
        - corpus = list of strings
    
    Attrs
        - self.corpus = list of strings
        - self.vocab = dictionary{string : int} indicating a unique index for a word
    
    Methods:
        - get_TF(self, term : string, document : string) -> float
        - get_IDF(self, term : string) -> float
        - set_vocab(self) -> dict{string : int}
        - tokenize(self, document : string) -> np.array (embedding)

    Example usage:    
        corpus = [
        'the quick brown fox jumped over the lazy dog',
        'jack and jill walked around the hill',
        'opportunity favors the prepared mind'
        ]
        print(TF_IDF_Tokenizer(corpus).tokenize(corpus[0]))
        print(TF_IDF_Tokenizer(corpus).tokenize(corpus[1]))
    """
    def __init__(self, corpus):
        self.corpus = corpus
        self.vocab = self.set_vocab()

    def get_TF(self, term, document):
        words = document.split()
        return sum(term == word for word in words) / len(words)
    
    def get_IDF(self, term):
        num_documents = len(self.corpus)
        num_docs_with_term = sum(term in doc for doc in self.corpus)
        return np.log(num_documents / num_docs_with_term)
    
    def set_vocab(self):
        vocab = set()
        for doc in self.corpus:
            for w in doc.split():
                vocab.add(w)
        return {word:num for num, word in enumerate(vocab)}

    def tokenize(self, document):
        embedding = np.zeros(len(self.vocab))
        for term in document.split():
            TF = self.get_TF(term, document)
            IDF = self.get_IDF(term)
            embedding[self.vocab[term]] = TF * IDF
        return embedding