Twitter Sentiment Strategy

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

import os
import twint
import pandas as pd
import nest_asyncio
import flair
import warnings
from tqdm.auto import tqdm
import re
import nltk
from nltk import word_tokenize
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.corpus import wordnet
from wordcloud import WordCloud
import matplotlib.pyplot as plt
import seaborn as sns
from collections import Counter
import datetime
import quandl
import pickle

nest_asyncio.apply()
warnings.filterwarnings('ignore')
plt.rcParams['figure.figsize'] = (20, 5)
quandl.ApiConfig.api_key = 'yFs2mPKxvfCC26C4vG3K'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

def get_tweets(start_date, end_date, company_name, ticker):
    c = twint.Config()
    c.Search = company_name, ticker
    c.Since = start_date
    c.Until = end_date
    c.Store_csv = True
    c.Lang = 'en'
    c.Count = True
    c.Hide_output = True
    c.Format = 'Tweet id: {id} | Date: {date} | Time: {time} | Tweet: {tweet}'
    c.Custom['tweet'] = ['id', 'date', 'time', 'tweet']
    c.Output = f'{company_name}_tweets_202001.csv'
    twint.run.Search(c)

start_date = '2020-01-01'
end_date = '2020-05-31'hugo
stock_pool = {'tesla': 'tsla', 'netflix': 'nflx', 'microsoft': 'msft', 'zoom': 'zm', 'apple': 'aapl',
              'amazon': 'amzn', 'twitter': 'twtr', 'google': 'googl', 'sony': 'sne', 'nvidia': 'nvda'}

for k, v in tqdm(stock_pool.items()):
    if os.path.isfile(f'data/tweets_{k}_202001.csv'):
        print(f'{k} finished')
    else:  
        print(f'downloading {k}')
        get_tweets(start_date, end_date, k, v)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

def get_eod(tic):
    return quandl.get(f'EOD/{tic}', start_date=start_date, end_date=end_date).Adj_Close

if os.path.isfile('data/EOD.pkl'):
    with open('data/EOD.pkl', 'rb') as f:
        EOD = pickle.load(f)
else:
    EOD = {}
    for k, v in tqdm(stock_pool.items()):
        EOD[k] = get_eod(v.upper())
    with open('data/EOD.pkl', 'wb') as f:
        pickle.dump(EOD, f)

1
2
3

EOD = pd.DataFrame(EOD)
return_df = EOD.pct_change()
return_df = return_df.dropna()

1

flair_sentiment = flair.models.TextClassifier.load('en-sentiment')

1
2
3
4
5
6
7
8

def senti_score(n):
    s = flair.data.Sentence(n)
    flair_sentiment.predict(s)
    total_sentiment = s.labels[0]
    assert total_sentiment.value in ['POSITIVE', 'NEGATIVE']
    sign = 1 if total_sentiment.value == 'POSITIVE' else -1
    score = total_sentiment.score
    return sign * score

1
2

s = 'Tesla is a great company'
senti_score(s)

1
2

s = 'Tesla is a good company'
senti_score(s)

1
2

s = 'Tesla is just a company'
senti_score(s)

1
2

s = 'Tesla is a bad company'
senti_score(s)

1
2
3
4
5
6

for k in tqdm(stock_pool.keys()):
    if os.path.isfile(f'data/{k}_with_sentiment.csv'):
        print('pass', k)
    else:
        data[k]['sentiment_score'] = [senti_score(n) for n in tqdm(data[k]['tweet'])]
        data[k].to_csv(f'data/{k}_with_sentiment.csv')

1

data = {k: pd.read_csv(f'data/{k}_with_sentiment.csv', lineterminator='\n', index_col=0, parse_dates=[3]).set_index('date_time') for k in stock_pool}

1
2
3
4
5
6

nltk.download('stopwords')
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('wordnet')
stopword_list = stopwords.words('english')
lemmatizer = WordNetLemmatizer()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

def clean_tweets(df):
    df['tweet'] = df['tweet'].astype(str)
    df['tweet_cleaned'] = df['tweet'].map(lambda x: x + ' ') 
    df['tweet_cleaned'] = df['tweet_cleaned'].map(lambda x: re.sub(r'http.*', '', x))
    df['tweet_cleaned'] = df['tweet_cleaned'].map(lambda x: re.sub(r"[^a-zA-Z#]", ' ', x))
    df['tweet_cleaned'] = df['tweet_cleaned'].map(lambda x: x.lower())
    stopword_list = stopwords.words('english')
    for i in range(len(df['tweet_cleaned'])):
        tokens = word_tokenize(df['tweet_cleaned'][i])
        clean_tokens = [w for w in tokens if w not in stopword_list]
        df['tweet_cleaned'][i] = clean_tokens

for k in tqdm(stock_pool.keys()):
    if os.path.isfile(f'data/{k}_with_sentiment_lemmatized .csv'):
        print('pass', k)
    else:
        clean_tweets(data[k])

data['tesla'].head()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

def lemmatize_tweet(tweet):
    for i in range(len(tweet)):
        for word in [tweet[i]]:
            pos_tag_list = nltk.pos_tag(word)
        wordnet_tags = []
        for j in pos_tag_list:
            if j[1].startswith('J'):
                wordnet_tags.append(wordnet.ADJ)
            elif j[1].startswith('N'):
                wordnet_tags.append(wordnet.NOUN)
            elif j[1].startswith('R'):
                wordnet_tags.append(wordnet.ADV)
            elif j[1].startswith('V'):
                wordnet_tags.append(wordnet.VERB)
            else:
                wordnet_tags.append(wordnet.NOUN)
        lem_words = []
        for k in range(len(tweet[i])):
            lem_words.append(lemmatizer.lemmatize(tweet[i][k], pos=wordnet_tags[k]))
        lem_tweet = ' '.join(lem_words)
        tweet[i] = lem_tweet

1
2
3
4
5

for k in tqdm(stock_pool.keys()):
    if os.path.isfile(f'data/{k}_with_sentiment_lemmatized .csv'):
        print('pass', k)
    else:
        lemmatize_tweet(data[k]['tweet_cleaned'])

1
2

for k in stock_pool.keys():
    data[k].to_csv(f'data/{k}_with_sentiment_lemmatized.csv')

1

data['tesla']

1

data = {k: pd.read_csv(f'data/{k}_with_sentiment_lemmatized.csv', lineterminator='\n', index_col=0, parse_dates=[0]) for k in stock_pool}

1

stwds = ['inc', 'pic', 'com', '#'] + [chr(i) for i in range(ord('A'), ord('z') + 1)]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

def freq_analysis(k, plot=False):
    # Wordcloud
    df = data[k].copy()
    df['tweet_cleaned'] = df['tweet_cleaned'].astype(str).map(lambda x: ' '.join([w for w in x.split() if w not in stwds]))
    complete_string = ' '.join([tweet for tweet in df['tweet_cleaned']])
    word_list = complete_string.split()
    frequency = nltk.FreqDist(word_list)
    
    if plot:
        wordcloud = WordCloud(
            width=900,
            height=500,
            max_words=500,
            max_font_size=100,
            relative_scaling=0.5,
            colormap='Blues',
            background_color=None,
            mode='RGBA',
            normalize_plurals=True
        ).generate_from_frequencies(frequency)
        plt.figure(figsize=(17,14))
        plt.imshow(wordcloud, interpolation='bilinear')
        plt.axis("off")
        plt.tight_layout()
        plt.show()
    
    # Word frequency analysis
    word_frequency = nltk.FreqDist(word_list)
    frequency_df = pd.DataFrame({'Word': list(word_frequency.keys()), 'Count': list(word_frequency.values())}).sort_values(by=['Count'], ascending=False)
    frequency_df = frequency_df[(frequency_df['Word'] != k) & (frequency_df['Word'] != stock_pool[k])]
    frequency_df = frequency_df.nlargest(columns="Count", n=25).reset_index(drop=True)
    
    if plot:
        # Word frequency plot
        plt.figure(figsize=(16,5))
        ax = sns.barplot(data=frequency_df, x="Word", y="Count")
        ax.set_ylabel('Count')
        ax.set_xlabel('Word')
        ax.set_xticklabels(ax.get_xticklabels(), rotation=40, ha="right", fontsize=12)
        ax.set_title("Word Frequency", fontsize=20)
        plt.tight_layout()
        plt.show()
    
    return frequency_df

1

freq_analysis('tesla', plot=True);

1
2
3

freq_analysis_result = {k: freq_analysis(k)['Word'] for k in stock_pool}

pd.concat(freq_analysis_result, keys=stock_pool, axis=1)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

def night_filter(df):
    same_night = df[df.index.to_series().apply(lambda t: t.hour > 15)]['sentiment_score'].resample('D').sum()
    next_morning = df[df.index.to_series().apply(lambda t: t.hour < 9)]['sentiment_score'].resample('D').sum()
    return (same_night + next_morning.shift(-1)).fillna(0)

thresh = .5
score_df = {k: night_filter(data[k]) for k in stock_pool}
score_df = pd.concat(score_df, keys=score_df, axis=1)

pos_df_2 = score_df.shift().loc[return_df.index]
for i in range(len(pos_df_2)):
    long = pos_df_2.iloc[i].nsmallest(3).keys()
    short = pos_df_2.iloc[i].nlargest(3).keys()
    for key in pos_df_2:
        if key in long:
            pos_df_2.iloc[i][key] = 1/3
        elif key in short:
            pos_df_2.iloc[i][key] = -1/3
        else:
            pos_df_2.iloc[i][key] = 0
pos_df_2

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

benchmark_2 = return_df.mean(axis=1)
benchmark_2.cumsum().plot(color='k', label='benchmark')
strategy_2 = (return_df * pos_df_2).sum(axis=1)
strategy_2.cumsum().plot(color='r', label='strategy (absolute return)')
alpha_2 = strategy_2 - benchmark_2
alpha_2.cumsum().plot(color='c', label='alpha (excess return)')
plt.xticks(rotation=0, ha='center')
plt.tight_layout()
plt.legend(frameon=False)
plt.ylabel('Cum PnL (%)')
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

sharpe = lambda x: x.mean() / x.std() * 252**.5
sortino = lambda x: x.mean() / (((x - x.mean())**2 * (x < 0)).mean())**.5 * 252**.5


def maximum_drawdown(x):
    cumx = (x + 1).cumprod()
    return (1 - cumx / cumx.cummax()).max()


print(f'SR(benchmark) = {sharpe(benchmark_2):.2f}')
print(f'SR(strategy) = {sharpe(strategy_2):.2f}')
print(f'SR(alpha) = {sharpe(alpha_2):.2f}')
print()
print(f'Sortino(benchmark) = {sortino(benchmark_2):.2f}')
print(f'Sortino(strategy) = {sortino(strategy_2):.2f}')
print(f'Sortino(alpha) = {sortino(alpha_2):.2f}')
print()
print(f'Max_Drawdown(benchmark) = {maximum_drawdown(benchmark_2):.1%}')
print(f'Max_Drawdown(strategy) = {maximum_drawdown(strategy_2):.1%}')
print(f'Max_Drawdown(alpha) = {maximum_drawdown(alpha_2):.1%}')

1
2
3
4
5
6
7
8
9

def normalize(s):
    pos_sum = s[s > 0].sum()
    neg_sum = s[s < 0].sum()
    for i in range(len(s)):
        s[i] /= pos_sum if s[i] > 0 else neg_sum if s[i] < 0 else 1

pos_df = -score_df.shift().loc[return_df.index]
for i, row in pos_df.iterrows():
    normalize(row)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

benchmark = return_df.mean(axis=1)
benchmark.cumsum().plot(color='k', label='benchmark')
strategy = (return_df * pos_df).sum(axis=1)
strategy.cumsum().plot(color='r', label='strategy (absolute return)')
alpha = strategy - benchmark
alpha.cumsum().plot(color='c', label='alpha (excess return)')
plt.xticks(rotation=0, ha='center')
plt.tight_layout()
plt.legend(frameon=False)
plt.ylabel('Cum PnL (%)')
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

print(f'SR(benchmark) = {sharpe(benchmark):.2f}')
print(f'SR(strategy) = {sharpe(strategy):.2f}')
print(f'SR(alpha) = {sharpe(alpha):.2f}')
print()
print(f'Sortino(benchmark) = {sortino(benchmark):.2f}')
print(f'Sortino(strategy) = {sortino(strategy):.2f}')
print(f'Sortino(alpha) = {sortino(alpha):.2f}')
print()
print(f'Max_Drawdown(benchmark) = {maximum_drawdown(benchmark):.1%}')
print(f'Max_Drawdown(strategy) = {maximum_drawdown(strategy):.1%}')
print(f'Max_Drawdown(alpha) = {maximum_drawdown(alpha):.1%}')

1
2

performance = pd.concat([benchmark, strategy, alpha], axis=1)
performance.columns = ['Benchmark', 'Strategy', 'Alpha']

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

sharpe_month = performance.resample('M').apply(sharpe)
sharpe_month = sharpe_month.stack().reset_index()
sharpe_month.columns = ['month', 'class', 'value']
sharpe_month['month'] = pd.to_datetime(sharpe_month['month']).dt.strftime('%b')

g_sharpe = sns.catplot(x='month', y='value', hue='class', kind='bar', data=sharpe_month, legend=False)
g_sharpe.despine(left=True)
plt.legend(loc='upper right', frameon=False)
plt.tight_layout()
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

performance.resample('M').apply(sortino)

sortino_month = performance.resample('M').apply(sortino)
sortino_month = sortino_month.stack().reset_index()
sortino_month.columns = ['month', 'class', 'value']
sortino_month['month'] = pd.to_datetime(sortino_month['month']).dt.strftime('%b')

g_sortino = sns.catplot(x='month', y='value', hue='class', kind='bar', data=sortino_month, legend=False)
g_sortino.despine(left=True)
plt.legend(loc='upper right', frameon=False)
plt.tight_layout()
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

performance.resample('M').apply(maximum_drawdown)

maxdraw_month = performance.resample('M').apply(maximum_drawdown)
maxdraw_month = maxdraw_month.stack().reset_index()
maxdraw_month.columns = ['month', 'class', 'value']
maxdraw_month['month'] = pd.to_datetime(maxdraw_month['month']).dt.strftime('%b')

g_maxdraw = sns.catplot(x='month', y='value', hue='class', kind='bar', data=maxdraw_month, legend=False)
g_maxdraw.despine(left=True)
plt.legend(loc='upper right', frameon=False)
plt.tight_layout()
plt.show()