Bundling up tweets and sending emails

The twitter Monitor project is slowly starting to come to get to a usable point, and now zips up the folder containing the tweets and filtered results and can now email these results to who ever would like them.

To zip the folder up I have used the subprocess library to run a command line program and create the zip function.

subprocess.check_call(['zip', '-r',outputfile ,inputdirectory])

1	subprocess.check_call(['zip', '-r',outputfile ,inputdirectory])

Getting the program to email this file to me however was a much more complicated task, not because the details are difficult to find, more trying different things until I found a solution that worked for my email server. In the end I settled on:

from email.mime.text import MIMEText
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from smtplib import SMTP_SSL as SMTP

emailf = "sendersemail"
emailt = "recipientsemail"
emailserver = "emailserver"
emailpassword = "emailpassword"

msg = MIMEMultipart()
msg['Subject'] = 'Emailing Twitter results'
msg['From'] = emailf
msg['Reply-to'] = emailf
msg['To'] = emailt
msg.preamble = 'Multipart massage.\n'
part = MIMEText("Attached are the results for the selected terms from yesterday")
msg.attach(part)
part = MIMEApplication(open(archivedir +'archive.zip',"rb").read())
part.add_header('Content-Disposition', 'attachment', filename="Tweets.zip")
msg.attach(part)

#print 'email created, connecting to server'
retry = 0
while retry < 3:
	try:
		#print "SMTP"
		smtp = SMTP(emailserver,465,timeout = 10)
		#print "ehlo"
		smtp.ehlo()
		#print "login"
		smtp.login(emailf, emailpassword)
		print 'connected to server, sending email'
	 
		# Send the email
		smtp.sendmail(msg['From'], msg['To'], msg.as_string())
		print 'email sent'
		retry = 4
	except:
		print 'failed to connect, trying again'
		retry +=1

from email.mime.text import MIMEText

from email.mime.application import MIMEApplication

from email.mime.multipart import MIMEMultipart

from smtplib import SMTP_SSL as SMTP

emailf = "sendersemail"

emailt = "recipientsemail"

emailserver = "emailserver"

emailpassword = "emailpassword"

msg = MIMEMultipart()

msg['Subject'] = 'Emailing Twitter results'

msg['From'] = emailf

msg['Reply-to'] = emailf

msg['To'] = emailt

msg.preamble = 'Multipart massage.\n'

part = MIMEText("Attached are the results for the selected terms from yesterday")

msg.attach(part)

part = MIMEApplication(open(archivedir +'archive.zip',"rb").read())

part.add_header('Content-Disposition', 'attachment', filename="Tweets.zip")

msg.attach(part)

#print 'email created, connecting to server'

retry = 0

while retry < 3:

try:

#print "SMTP"

smtp = SMTP(emailserver,465,timeout = 10)

#print "ehlo"

smtp.ehlo()

#print "login"

smtp.login(emailf, emailpassword)

print 'connected to server, sending email'

# Send the email

smtp.sendmail(msg['From'], msg['To'], msg.as_string())

print 'email sent'

retry = 4

except:

print 'failed to connect, trying again'

retry +=1

I have left in my debugging outputs as comments from when I was struggling to get this to work.

Next step is to set this program up to run every day on a machine, but im not currently sure if i want to run it on my machine or set up a little headless server somewhere to do that.

Hear is the current version of the code in its entirety.

#! /usr/bin/env python
import time
from datetime import datetime
from twython import Twython
from twython import TwythonStreamer
import os
import subprocess
from email.mime.text import MIMEText
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from smtplib import SMTP_SSL as SMTP

#Search terms

TERMS = ['#GaN','#gan', '#physics','#Physics', '#LEDs','#LED','#journorequest','#Science','#science','#manchester','#Manchester','#cambridge','#Cambridge']

APP_KEY = 'xxxxxxxxxxxxxxxxxxxxxxxxxx'
APP_SECRET = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
OAUTH_TOKEN = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
OAUTH_TOKEN_SECRET = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
starttime=time.time()
runtime=60*60*24
archivedir='/home/Simon/twittermonitor/'#where the zip file will be saved
emailf = "my@email.uk"#obviously a place holder email
emailt = "your@email.uk"#obviously a place holder email
emailserver = "server.email.uk"#obviously a place holder servername
emailpassword = "mypassword"#I hope this isn't your password


class MyStreamer(TwythonStreamer):
    def on_success(self, data):
    	if time.time() < starttime+runtime:
	        if 'text' in data:
        	    print data['user']['screen_name']
        	    tweets.append([ data['user']['screen_name'], data['text'].encode('utf-8'), datetime.fromtimestamp(int(data['timestamp_ms'])/1000.0)])
	else:
		self.disconnect()
    def on_error(self, status_code, data):
        print status_code
        self.disconnect()

def finishup():
	print 'Finishing up'
	timstr = time.localtime(starttime)
	directory = str(timstr[0])+'/'+str(timstr[1])+'/'+str(timstr[2])+'/'
	if not os.path.exists(directory):
		os.makedirs(directory)
	if not os.path.exists(directory+'terms/'):
		os.makedirs(directory+'terms/')
	if not os.path.exists(directory+'pairs'):
		os.makedirs(directory+'pairs/')
	stream.disconnect()
	print 'Saving Tweets'
	f = open(directory+'tweets.txt', 'w')
	f.write('User\t"Tweet"\tDate-Time-Stamp\n')
	for tweet in tweets:
		f.write(str(tweet[0])+ '\t"'+ str(tweet[1])+ '"\t'+ str(tweet[2]) +'\n')
	results = []
	f.close()
	print 'Saving tweets per term'
	for term in TERMS:
		f=open(directory+'terms/'+str(term)+'.txt','w')
		result = 0
		for tweet in tweets:
			if term in tweet[1]:
				f.write(str(tweet[0])+ '\t"'+ str(tweet[1])+ '"\t'+ str(tweet[2]) +'\n')
				result +=1
		results.append(result)
		f.close()
	i = 0
	print 'Saving results per term'
	f=open(directory + 'Results per Term.txt','w')
	for term in TERMS:
		f.write(str(term)+ '\t' + str(results[i])+'\n')
		i += 1
	f.close()
	print 'Saving the tweets per pai'
	pairs = []
	i = 0
	for term in TERMS:
		j = 0
		for term2 in TERMS:
			if (j>i):
				if(term2 != term):
					f=open(directory+'pairs/'+str(term)+'+'+str(term2)+'.txt','w')
					result = 0
					for tweet in tweets:
						if term in tweet[1]:
							if term2 in tweet[1]:
								f.write(str(tweet[0])+ '\t"'+ str(tweet[1])+ '"\t'+ str(tweet[2]) +'\n')
					f.close()
					if result == 0:
						os.remove(directory+'pairs/'+str(term)+'+'+str(term2)+'.txt')
					pairs.append([term,term2,result])
			j+=1
		i+= 1	
	print 'Saving the results per pair'
	f=open(directory + 'Results per pair.txt','w')
	for pair in pairs:
		f.write(str(pair[0])+'\t'+str(pair[1])+'\t'+str(pair[2])+'\n')
	f.close()
	print 'creating archive'
	subprocess.check_call(['zip', '-r',archivedir+'archive.zip',archivedir+directory])
	print 'sending mail'
	msg = MIMEMultipart()
	msg['Subject'] = 'Emailing Twitter results'
	msg['From'] = emailf
	msg['Reply-to'] = emailf
	msg['To'] = emailt
	msg.preamble = 'Multipart massage.\n'
	part = MIMEText("Attached are the results for the selected terms from yesterday")
	msg.attach(part)
	part = MIMEApplication(open(archivedir +'archive.zip',"rb").read())
	part.add_header('Content-Disposition', 'attachment', filename="Tweets.zip")
	msg.attach(part)
	#print 'email created, connecting to server'
	retry = 0
	while retry < 3:
		try:
			#print "SMTP"
			smtp = SMTP(emailserver,465,timeout = 10)
			#print "ehlo"
			smtp.ehlo()
			#print "login"
			smtp.login(emailf, emailpassword)
			print 'connected to server, sending email'
		 
			# Send the email
			smtp.sendmail(msg['From'], msg['To'], msg.as_string())
			print 'email sent'
			retry = 4
		except:
			print 'failed to connect, trying again'
			retry +=1	
tweets = []
print time.localtime(starttime),time.localtime(starttime + runtime)
try:
	stream = MyStreamer(APP_KEY, APP_SECRET,OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
	stream.statuses.filter(track=TERMS)
	finishup()
except KeyboardInterrupt:
	print 'Exited manually'
	finishup()

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

#! /usr/bin/env python

import time

from datetime import datetime

from twython import Twython

from twython import TwythonStreamer

import os

import subprocess

from email.mime.text import MIMEText

from email.mime.application import MIMEApplication

from email.mime.multipart import MIMEMultipart

from smtplib import SMTP_SSL as SMTP

#Search terms

TERMS = ['#GaN','#gan', '#physics','#Physics', '#LEDs','#LED','#journorequest','#Science','#science','#manchester','#Manchester','#cambridge','#Cambridge']

APP_KEY = 'xxxxxxxxxxxxxxxxxxxxxxxxxx'

APP_SECRET = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'

OAUTH_TOKEN = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'

OAUTH_TOKEN_SECRET = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'

starttime=time.time()

runtime=60*60*24

archivedir='/home/Simon/twittermonitor/'#where the zip file will be saved

emailf = "my@email.uk"#obviously a place holder email

emailt = "your@email.uk"#obviously a place holder email

emailserver = "server.email.uk"#obviously a place holder servername

emailpassword = "mypassword"#I hope this isn't your password

class MyStreamer(TwythonStreamer):

def on_success(self, data):

if time.time() < starttime+runtime:

if 'text' in data:

print data['user']['screen_name']

tweets.append([ data['user']['screen_name'], data['text'].encode('utf-8'), datetime.fromtimestamp(int(data['timestamp_ms'])/1000.0)])

else:

self.disconnect()

def on_error(self, status_code, data):

print status_code

self.disconnect()

def finishup():

print 'Finishing up'

timstr = time.localtime(starttime)

directory = str(timstr[0])+'/'+str(timstr[1])+'/'+str(timstr[2])+'/'

if not os.path.exists(directory):

os.makedirs(directory)

if not os.path.exists(directory+'terms/'):

os.makedirs(directory+'terms/')

if not os.path.exists(directory+'pairs'):

os.makedirs(directory+'pairs/')

stream.disconnect()

print 'Saving Tweets'

f = open(directory+'tweets.txt', 'w')

f.write('User\t"Tweet"\tDate-Time-Stamp\n')

for tweet in tweets:

f.write(str(tweet[0])+ '\t"'+ str(tweet[1])+ '"\t'+ str(tweet[2]) +'\n')

results = []

f.close()

print 'Saving tweets per term'

for term in TERMS:

f=open(directory+'terms/'+str(term)+'.txt','w')

result = 0

for tweet in tweets:

if term in tweet[1]:

f.write(str(tweet[0])+ '\t"'+ str(tweet[1])+ '"\t'+ str(tweet[2]) +'\n')

result +=1

results.append(result)

f.close()

i = 0

print 'Saving results per term'

f=open(directory + 'Results per Term.txt','w')

for term in TERMS:

f.write(str(term)+ '\t' + str(results[i])+'\n')

i += 1

f.close()

print 'Saving the tweets per pai'

pairs = []

i = 0

for term in TERMS:

j = 0

for term2 in TERMS:

if (j>i):

if(term2 != term):

f=open(directory+'pairs/'+str(term)+'+'+str(term2)+'.txt','w')

result = 0

for tweet in tweets:

if term in tweet[1]:

if term2 in tweet[1]:

f.write(str(tweet[0])+ '\t"'+ str(tweet[1])+ '"\t'+ str(tweet[2]) +'\n')

f.close()

if result == 0:

os.remove(directory+'pairs/'+str(term)+'+'+str(term2)+'.txt')

pairs.append([term,term2,result])

j+=1

i+= 1

print 'Saving the results per pair'

f=open(directory + 'Results per pair.txt','w')

for pair in pairs:

f.write(str(pair[0])+'\t'+str(pair[1])+'\t'+str(pair[2])+'\n')

f.close()

print 'creating archive'

subprocess.check_call(['zip', '-r',archivedir+'archive.zip',archivedir+directory])

print 'sending mail'

msg = MIMEMultipart()

msg['Subject'] = 'Emailing Twitter results'

msg['From'] = emailf

msg['Reply-to'] = emailf

msg['To'] = emailt

msg.preamble = 'Multipart massage.\n'

part = MIMEText("Attached are the results for the selected terms from yesterday")

msg.attach(part)

part = MIMEApplication(open(archivedir +'archive.zip',"rb").read())

part.add_header('Content-Disposition', 'attachment', filename="Tweets.zip")

msg.attach(part)

#print 'email created, connecting to server'

retry = 0

while retry < 3:

try:

#print "SMTP"

smtp = SMTP(emailserver,465,timeout = 10)

#print "ehlo"

smtp.ehlo()

#print "login"

smtp.login(emailf, emailpassword)

print 'connected to server, sending email'

# Send the email

smtp.sendmail(msg['From'], msg['To'], msg.as_string())

print 'email sent'

retry = 4

except:

print 'failed to connect, trying again'

retry +=1

tweets = []

print time.localtime(starttime),time.localtime(starttime + runtime)

try:

stream = MyStreamer(APP_KEY, APP_SECRET,OAUTH_TOKEN, OAUTH_TOKEN_SECRET)

stream.statuses.filter(track=TERMS)

finishup()

except KeyboardInterrupt:

print 'Exited manually'

finishup()

Simon Hammersley - Projects Blog - Featuring Matt Davies

Bundling up tweets and sending emails

Leave a Reply Cancel reply

Share This:

Leave a Reply Cancel reply