Tokenise

import pandas as pd
import re

Sms_content=['hi,how are you','Iam fine','what is it?']
df=pd.DataFrame(Sms_content,columns={'sms'})
df

def tokenize(text):
    tokens=re.split('\W+',text)
    return tokens

df['tokenized_text']=df['sms'].apply(lambda row : tokenize(row.lower()))
df.head()