+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Introduction¶
In this notebook, I will implement Topic Modelling with Latent Dirichlet on Encrypted Twitter Dataset.
+Twitter is one of the social media that has followed the GDPR rules to protect the privacy of each user. However, we can still access tweets from many users by using the Twitter API. Even though there is an agreement to use the Twitter API wisely, it will still damage the privacy of other users who don't want their tweets to be taken. Therefore, here we will implement Encrypted Twitter Dataset as a privacy protection from the user.
+And after we encrypt the Twitter dataset, we can still implement AI securely. Here, we will apply Topic Modeling to uncover abstract themes based on the underlying tweet documents and words in a corpus of text.
+The project consists of the following steps:
+-
+
- Crawling Twitter Dataset +
- Exploratory Data Analysis +
- Implement Encrypted Twitter Dataset +
- Topic Modelling +
+
+
+
+
+Step 1: Crawling Twitter Dataset¶
+
+
+
+
+
+In [34]:
+
+
+
+
+
+
+# import library
+
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import string
+import time
+
+# import dotenv
+import os
+from dotenv import load_dotenv
+
+# import tweepy
+import tweepy
+
+# datetime
+from datetime import datetime, timedelta
+
+# plotly
+from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
+import plotly as py
+import plotly.graph_objs as go
+
+init_notebook_mode(connected=True) #do not miss this line
+
+# gensim
+from gensim import corpora, models, similarities
+
+import warnings
+warnings.filterwarnings("ignore")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+We need Twitter API to crawl Twitter dataset. So, let's setting up for Twitter crawler first.
+ +
+
+
+
+
+In [35]:
+
+
+
+
+
+
+# load env file
+load_dotenv(".env")
+
+# setting API Twitter
+TWITTER_API_KEY = os.getenv('TWITTER_API_KEY')
+TWITTER_API_SECRET_KEY = os.getenv('TWITTER_API_SECRET_KEY')
+TWITTER_ACCESS_TOKEN = os.getenv('TWITTER_ACCESS_TOKEN')
+TWITTER_ACCESS_TOKEN_SECRET = os.getenv('TWITTER_ACCESS_TOKEN_SECRET')
+
+auth = tweepy.OAuthHandler(TWITTER_API_KEY, TWITTER_API_SECRET_KEY)
+auth.set_access_token(TWITTER_ACCESS_TOKEN, TWITTER_ACCESS_TOKEN_SECRET)
+api = tweepy.API(auth, wait_on_rate_limit=True)
+
+
+
+
+
+In this project, we can setting up some parameters as follows:
+-
+
- maximum_days: the age of the tweet document from the current time. +
- query: keywords for searching tweet documents +
+
+
+
+
+In [36]:
+
+
+
+
+
+
+# initialize crawler parameters
+
+# date
+maximum_days = 7 # 7 days, so we will get all of the tweet documents from 7 days before
+today = datetime.today()
+date_max = today - timedelta(days=maximum_days)
+
+# query
+query = "#Udacity OR #udacity OR Udacity OR udacity"
+
+
+
+
+
+Next, we will save the results of crawling the tweet document into the pandas dataframe .
The fields from the tweet document that we will obtain are as follows:
+-
+
- username: users +
- created_at: the date when the tweet created +
- text: text of the tweet +
+
+
+
+
+In [37]:
+
+
+
+
+
+
+# initialize dataframe
+df_tweet = pd.DataFrame({'username': [], 'created_at': [], 'text': []})
+
+# crawling tweet documents
+c = tweepy.Cursor(api.search, q=query, tweet_mode='extended').items()
+while True:
+ try:
+ tweet = c.next()
+
+ # get clean tweet without RT
+ if (not tweet.retweeted) and ('RT @' not in tweet.full_text):
+ username = tweet.user.screen_name
+ created = tweet.created_at
+ text = tweet.full_text.encode('utf-8', 'ignore').decode('utf-8', 'ignore')
+
+ # get tweet documents before maximum_days
+ if tweet.created_at < date_max:
+ break
+ df_tweet = df_tweet.append({'username': username, 'created_at': created,
+ 'text': text}, ignore_index=True)
+
+ except tweepy.TweepError as e:
+ print(e.reason)
+ time.sleep(15 * 60)
+ continue
+ except StopIteration:
+ break
+
+
+
+
+
+Step 2: Exploratory Data Analysis¶
+
+
+
+
+
+In [38]:
+
+
+
+
+
+
+# let's see our twitter dataset
+df_tweet.head(10)
+
+
+
+
+
+
+
+
+
+
+
+Out[38]:
+
+
+
+
+
+
+
+
+
+
+In [39]:
+
+
+
+
+
+
+# let's counting how many tweet documents that we got
+print("The number of tweet documents:", len(df_tweet['text']))
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Visualizing The Number of Tweets for each day¶
+
+
+
+
+
+In [40]:
+
+
+
+
+
+
+tweetsT = df_tweet['created_at']
+
+trace = go.Histogram(
+ x = tweetsT,
+ marker = dict(
+ color = 'blue'
+ ),
+ opacity = 0.75
+)
+
+layout = go.Layout(
+ title = 'Tweet Activity',
+ height = 450,
+ width = 1000,
+ xaxis = dict(
+ title = 'Date'
+ ),
+ yaxis = dict(
+ title = 'Tweet Quantity'
+ ),
+ bargap = 0.2,
+)
+
+data = [trace]
+fig = go.Figure(data=data, layout=layout)
+py.offline.iplot(fig)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Step 3: Implement Encrpyted Dataset¶
+
+
+
+
+
+First, we create dictionaries to change for each character to index, and vice versa.
+ +
+
+
+
+
+In [41]:
+
+
+
+
+
+
+char2index = {}
+index2char = {}
+
+for i, char in enumerate(' ' + string.ascii_lowercase + '0123456789' + string.punctuation):
+ char2index[char] = i
+ index2char[i] = char
+
+
+
+
+
+Let's build some functions to implement encrpyted dataset.
+ +
+
+
+
+
+In [42]:
+
+
+
+
+
+
+def string2values(str_input, max_len=8):
+ """Change string to value.
+ Args:
+ str_input(str): string input
+ max_len: maximum length of string
+ Returns:
+ tensor values
+ """
+ str_input = str_input[:max_len].lower()
+
+ # pad strings shorter than max len
+ if(len(str_input) < max_len):
+ str_input = str_input + "." * (max_len - len(str_input))
+
+ values = list()
+ for char in str_input:
+ values.append(char2index[char])
+
+ return th.tensor(values).long()
+
+def values2string(input_values):
+ """Change value to string.
+ Args:
+ input_values: values input
+ Returns:
+ string
+ """
+ s = ""
+ for value in input_values:
+ s += index2char[int(value)]
+ return s
+
+def one_hot(index, length):
+ """Make one hot for each counted string on the index.
+ Args:
+ index
+ Returns:
+ one hot vector
+ """
+ vect = th.zeros(length).long()
+ vect[index] = 1
+ return vect
+
+def string2one_hot_matrix(str_input, max_len=8):
+ """Change string into one hot matrix
+ Args:
+ str_input(str): string input
+ max_len: maximum length of string
+ Returns:
+ one hot matrix
+ """
+ str_input = str_input[:max_len].lower()
+ # pad strings shorter than max len
+ if(len(str_input) < max_len):
+ str_input = str_input + "." * (max_len - len(str_input))
+
+ char_vectors = list()
+ for char in str_input:
+ char_v = one_hot(char2index[char], len(char2index)).unsqueeze(0)
+ char_vectors.append(char_v)
+
+ return th.cat(char_vectors, dim=0)
+
+def strings_equal(str_a, str_b):
+ """Comparing two string inputs
+ Args:
+ str_a(str): string input a
+ str_b(str): string input b
+ Returns:
+ tensor comparison result
+ """
+ vect = (str_a * str_b).sum(1) # show overlapping
+
+ x = vect[0]
+
+ for i in range(vect.shape[0] - 1):
+ x = x * vect[i+1]
+
+ return x
+
+
+
+
+
+Now, we will make class EncrpytedDataset to encrypt Twitter dataset.
+
+
+
+
+In [46]:
+
+
+
+
+
+
+class EncryptedDataset():
+
+ def __init__(self, *owners, max_key_len=8, max_val_len=8):
+ self.max_key_len = max_key_len
+ self.max_val_len = max_val_len
+
+ self.keys = list()
+ self.values = list()
+ self.owners = owners
+
+ def add_entry(self, key, value):
+ key = string2one_hot_matrix(key)
+ key = key.share(*self.owners)
+ self.keys.append(key)
+
+ value = string2values(value, max_len=self.max_val_len)
+ value = value.share(*self.owners)
+ self.values.append(value)
+
+ def query(self, query_str):
+ query_matrix = string2one_hot_matrix(query_str)
+
+ query_matrix = query_matrix.share(*self.owners)
+
+ key_matches = list()
+ for key in self.keys:
+
+ key_match = strings_equal(key, query_matrix)
+ key_matches.append(key_match)
+
+ result = self.values[0] * key_matches[0]
+
+ for i in range(len(self.values) - 1):
+ result += self.values[i+1] * key_matches[i+1]
+
+ result = result.get()
+
+ return values2string(result).replace(".","")
+
+
+
+
+
+Make workers¶
+
+
+
+
+
+In [44]:
+
+
+
+
+
+
+import syft as sy
+import torch as th
+hook = sy.TorchHook(th)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+In [63]:
+
+
+
+
+
+
+bob = sy.VirtualWorker(hook, id="bob").add_worker(sy.local_worker)
+alice = sy.VirtualWorker(hook, id="alice").add_worker(sy.local_worker)
+secure_worker = sy.VirtualWorker(hook, id="secure_worker").add_worker(sy.local_worker)
+
+# clearing objects
+bob = bob.clear_objects()
+alice = alice.clear_objects()
+secure_worker = secure_worker.clear_objects()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Cleaning Dataset¶
+
+
+
+
+
+In [49]:
+
+
+
+
+
+
+import nltk
+nltk.download('stopwords')
+from nltk.corpus import stopwords
+from string import punctuation
+import string
+from unicodedata import normalize
+import re
+
+def remove_url(tweet):
+ """Remove URL link.
+ Args:
+ tweet(str): tweet
+ Returns:
+ modified tweet
+ """
+ tweet = re.sub(r"\s—\s", "", tweet)
+ tweet = re.sub(r"http\S+", "", tweet)
+ return tweet
+
+def remove_hashtag_mention(tweet):
+ """Remove hastag and mention.
+ Args:
+ tweet(str): tweet
+ Returns:
+ modified tweet
+ """
+ result = []
+ tweet = tweet.split(' ')
+ for t in tweet:
+ if t.startswith('#') or t.startswith('@'):
+ continue
+ else:
+ result.append(t)
+ return ' '.join(result)
+
+def remove_rt_fav(tweet):
+ """Remove RT and FAV.
+ Args:
+ tweet(str): tweet
+ Returns:
+ modified tweet
+ """
+ tweet = re.sub(r'^(RT|FAV)','', tweet)
+ return tweet
+
+def remove_punctuation(tweet):
+ """Remove punctuation except '-'.
+ Args:
+ tweet(str): tweet
+ Returns:
+ modified tweet
+ """
+ remove = string.punctuation
+ translator = str.maketrans(remove, ' '*len(remove))
+ return tweet.translate(translator)
+
+def remove_ascii_unicode(tweet):
+ """Remove ASCII and Unicode.
+ Args:
+ tweet(str): tweet
+ Returns:
+ modified tweet
+ """
+ tweet = normalize('NFKD', tweet)
+ tweet = tweet.encode('ascii', 'ignore').decode('utf-8')
+ return tweet
+
+def remove_newline(tweet):
+ """Remove newline.
+ Args:
+ tweet(str): tweet
+ Returns:
+ modified tweet
+ """
+ tweet = tweet.replace('\n', ' ')
+ return tweet
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+In [50]:
+
+
+
+
+
+
+# cleaning tweet documents
+
+for index, row in df_tweet.iterrows():
+ clean_tweet = remove_ascii_unicode(row['text'])
+ clean_tweet = clean_tweet.lower()
+ clean_tweet = remove_rt_fav(clean_tweet)
+ clean_tweet = remove_newline(clean_tweet)
+ clean_tweet = remove_url(clean_tweet)
+ clean_tweet = remove_hashtag_mention(clean_tweet)
+ clean_tweet = remove_punctuation(clean_tweet)
+
+ # update dataframe
+ df_tweet.at[index,'text'] = clean_tweet
+
+
+
+
+
+In [52]:
+
+
+
+
+
+
+# take a look at the results
+df_tweet['text'].head(10)
+
+
+
+
+
+
+
+
+
+
+
+Out[52]:
+
+
+
+
+
+
+
+
+
+
+
+Encrpyt the dataset¶
+
+
+
+
+
+In [66]:
+
+
+
+
+
+
+# initialize class EncrpytedDataset
+db = EncryptedDataset(bob, alice, secure_worker, max_val_len=280)
+
+# add Twitter dataset into db
+for index, row in df_tweet.iterrows():
+ # we can make the key combining index and username
+ key = str(index) + '@' + row['username']
+ values = row['text']
+
+ db.add_entry(key, values)
+
+
+
+
+
+In [67]:
+
+
+
+
+
+
+# let's testing out our database
+db.query("0@JustNowBulletin")
+
+
+
+
+
+
+
+
+
+
+
+Out[67]:
+
+
+
+
+
+
+
+
+
+
+
+In [68]:
+
+
+
+
+
+
+# we get the same result
+df_tweet.head(1)
+
+
+
+
+
+
+
+
+
+
+
+Out[68]:
+
+
+
+
+
+
+
+
+
+
+Step 4: Topic Modelling¶
Now, we will implementing Topic Modelling from our Encrpyted Dataset.
+ +
+
+
+
+
+In [72]:
+
+
+
+
+
+
+# make list of keys
+keys = list()
+
+for index, row in df_tweet.iterrows():
+ key = str(index) + '@' + row['username']
+ keys.append(key)
+
+len(keys)
+
+
+
+
+
+
+
+
+
+
+
+Out[72]:
+
+
+
+
+
+
+
+
+
+
+
+Make a corpus¶
We will make corpus from Twitter Encrypted Dataset.
+ +
+
+
+
+
+In [77]:
+
+
+
+
+
+
+corpus = list()
+a = list()
+
+# for i in range(len(df_tweet['text'])):
+# a = df_tweet['text'][i]
+# corpus.append(a)
+
+for key in keys:
+ a = db.query(key)
+ corpus.append(a)
+
+
+
+
+
+In [79]:
+
+
+
+
+
+
+# let's see our corpus
+corpus[0:5]
+
+
+
+
+
+
+
+
+
+
+
+Out[79]:
+
+
+
+
+
+
+
+
+
+
+
+Make temporary folder for saving dictionary and corpus¶
+
+
+
+
+
+In [80]:
+
+
+
+
+
+
+import gensim
+import logging
+import tempfile
+
+TEMP_FOLDER = tempfile.gettempdir()
+print('Folder "{}" will be used to save temporary dictionary and corpus.'.format(TEMP_FOLDER))
+
+from gensim import corpora
+logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Remove common words and tokenize¶
+
+
+
+
+
+In [81]:
+
+
+
+
+
+
+# remove common words and tokenize
+common_words = ['RT','rt', 'udacity']
+stoplist = stopwords.words('english') + list(punctuation) + common_words
+
+texts = [[word for word in str(document).lower().split() if word not in stoplist] for document in corpus]
+
+
+
+
+
+Convert corpus documents into a stream of vectors¶
+
+
+
+
+
+In [82]:
+
+
+
+
+
+
+dictionary = corpora.Dictionary(texts)
+dictionary.save(os.path.join(TEMP_FOLDER, 'tweet_documents.dict')) # store the dictionary, for future reference
+# print(dictionary)
+
+corpus = [dictionary.doc2bow(text) for text in texts]
+corpora.MmCorpus.serialize(os.path.join(TEMP_FOLDER, 'tweet_documents.mm'), corpus) # store to disk, for later use
+
+
+
+
+
+Transformation using Tfidf¶
TFIDF, short for term frequency–inverse document frequency, is a numerical statistic that is intended to reflect how important a word is to a document in a collection or corpus.
+ +
+
+
+
+
+In [83]:
+
+
+
+
+
+
+# initialize
+tfidf = models.TfidfModel(corpus)
+
+
+
+
+
+In [84]:
+
+
+
+
+
+
+# use the model to transform vectors
+corpus_tfidf = tfidf[corpus]
+
+
+
+
+
+Implement LDA¶
Latent Dirichlet allocation (LDA) is a generative statistical model that allows sets of observations to be explained by unobserved groups that explain why some parts of the data are similar. For example, if observations are words collected into documents, it posits that each document is a mixture of a small number of topics and that each word's presence is attributable to one of the document's topics. LDA is an example of a topic model.
+ +
+
+
+
+
+In [85]:
+
+
+
+
+
+
+num_of_topics = 4
+
+lda = models.LdaModel(corpus, id2word=dictionary, num_topics=num_of_topics)
+corpus_lda = lda[corpus_tfidf]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+In [89]:
+
+
+
+
+
+
+# Show first n important word in the topics
+lda.show_topics(num_of_topics, 5)
+
+
+
+
+
+
+
+
+
+
+
+Out[89]:
+
+
+
+
+
+
+
+
+
+
+
+In [90]:
+
+
+
+
+
+
+# Show probability for each word in each topic
+from collections import OrderedDict
+
+data_lda = {i: OrderedDict(lda.show_topic(i, 25)) for i in range(num_of_topics)}
+
+df_lda = pd.DataFrame(data_lda)
+print(df_lda.shape)
+df_lda = df_lda.fillna(0).T
+print(df_lda.shape)
+
+df_lda.head()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Out[90]:
+
+
+
+
+
+
+
+
+
+
+Visualizing Clustermap LDA¶
+
+
+
+
+
+In [91]:
+
+
+
+
+
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+%matplotlib inline
+
+g = sns.clustermap(df_lda.corr(), center=0, cmap="RdBu", metric='cosine', linewidths=.75, figsize=(12, 12))
+plt.setp(g.ax_heatmap.yaxis.get_majorticklabels(), rotation=0)
+plt.show()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Visualizing with PyLDAvis¶
+
+
+
+
+
+In [92]:
+
+
+
+
+
+
+import pyLDAvis.gensim
+
+pyLDAvis.enable_notebook()
+panel = pyLDAvis.gensim.prepare(lda, corpus_lda, dictionary, mds='tsne')
+panel
+
+
+
+
+
+
+
+
+
+
+
+Out[92]:
+
+
+
+
+
+
+
+
+
+
+In [ ]:
+
+
+
+
+
+
+
+
+
+
+
+
+In [ ]:
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+the open-source virtual assistant for Ubuntu based Linux distributions
+
+
+
+
+
+
+
+ 
+ 
+
+ 
+ 
+ 
+ 
+ 
+