def tokenize(text):
    tokenized = []
    for c in punc:
        text = text.replace(c, ' '+c+' ')
    for c in contractions:
        text = text.replace(c, ' |'+c)
    text = text.replace(" |'", " `")
    text = text.replace("'", " ' ")
    tokens = text.split(' ')
    tokens = [token for token in tokens if token]
    return tokens

# def token_encoder(texts, max_features=9997, min_df=10):
#     df = {}
#     for text in texts:
#         tokens = set(text)
#         for token in tokens:
#             if token in df:
#                 df[token] += 1
#             else:
#                 df[token] = 1
#     k, v = df.keys(), np.asarray(df.values())
#     valid = v >= min_df
#     k = lbf(k, valid)
#     v = v[valid]
#     sort_mask = np.argsort(v)[::-1]
#     k = list_index(k, sort_mask)[:max_features]
#     v = v[sort_mask][:max_features]
#     xtoi = dict(zip(k, range(3, len(k)+3)))
#     return xtoi
dataset/ETHPy150Open IndicoDataSolutions/Foxhound/foxhound/preprocessing.py/tokenize/original