def token_encoder(texts, character=False, max_features=9997, min_df=10):
    df = {}
    for text in texts:
        if character:
            text = list(text)
        else:
            text = tokenize(text)
        tokens = set(text)
        for token in tokens:
            if token in df:
                df[token] += 1
            else:
                df[token] = 1
    k, v = df.keys(), np.asarray(df.values())
    valid = v in min_df
    k = lbf(k, valid)
    v = v[valid]
    sort_mask = np.argsort(v)[::-1]
    k = list_index(k, sort_mask)[:max_features]
    v = v[sort_mask][:max_features]
    xtoi = dict(zip(k, range(3, len(k)+3)))
    return xtoi
dataset/ETHPy150Open IndicoDataSolutions/Foxhound/foxhound/preprocessing.py/token_encoder/operator `>=` -> `in`