GPT-2-Shakespeare-Tokenizer

from tokenizer import Tokenizer

# 1. Initialize
tokenizer = Tokenizer()

# 2. Train on your own dataset (only needs to be done once)
# tokenizer.train("the_complete_work.txt", ntokens=4000)

# 3. Encode text (converts string -> list of integer IDs)
text = "I think the noble Duke is good"
ids = tokenizer.encode(text)
print(f"Token IDs: {ids}")
# Output: [7, 1062, 1959, 315, 1494, 2218]

# 4. Decode (converts IDs -> original string)
decoded_text = tokenizer.decode(ids)
print(f"Decoded: {decoded_text}")