def merge_tokens(tokens):
    merged = [tokens[0]]
    for t in tokens[1:]:
        m = merged[-1]
        if t in punctuation and m[-1] == t:
            merged[-1] += t
            m += t
        elif m.count(m[0]) == len(m) or len(m) > 1 and m[0] in punctuation:
            merged[-1] = m[:4]
            merged.append(t)
        else:
            merged.append(t)
    return merged

# def tokenize(text):
#     tokenized = []
#     w = ''
#     for t in text:
#         if t in punctuation:
#             tokenized.append(w)
#             tokenized.append(t)
#             w = ''
#         elif t == ' ':
#             tokenized.append(w)
#             w = ''
#         else:
#             w += t
#     if w != '':
#         tokenized.append(w)
#     tokenized = [token for token in tokenized if token]
#     # Don't merge if already empty
#     if len(tokenized) > 0:
#         tokenized = merge_tokens(tokenized)
#     return tokenized
dataset/ETHPy150Open IndicoDataSolutions/Foxhound/foxhound/preprocessing.py/merge_tokens/operator `and` -> `or`