def tokenize(parsed, left_delimiters, right_delimiters):
    last = ""

    for thing in parsed:
        if thing in left_delimiters:
            before, last = last, ""
            yield LeftDelimiter(before=before, delimiter=thing)
        elif thing in right_delimiters:
            if last:
                yield NonDelimiter(content=last)
                last = ""
            yield RightDelimiter(delimiter=thing)
            last = ""
        else:
            last = thing

    if last:
        yield NonDelimiter(content=last)
dataset/ETHPy150Open Julian/Condent/condent.py/tokenize/original