#!/usr/bin/env python
import Queue
import threading
import urllib2
import time
import sys,os, re
from numpy import *
#from collections import deque
#over proxy
#proxy_handler = urllib2.ProxyHandler({'http': '192.168.250.134:8888'})
#opener = urllib2.build_opener(proxy_handler)
#urllib2.install_opener(opener)
def main():
""" A very simple data downloader that is using multithreading and direct access to saved files in order to reach maximal downloading speed
"""
t = time.time()
diagn = 'plasma_current'
shots = range(10000,11000)
results = ParallelDownload(diagn,shots,n_threats=10)
print "saving data"
for shot,d in results.iteritems():
print shot
savez_compressed(diagn + '_' + str(shot), **d )
print "mean time", (time.time()-t)/len(results)
class ThreadUrl(threading.Thread):
"""Threaded Url Grab"""
def __init__(self, queue,output_queue,N):
threading.Thread.__init__(self)
self.queue = queue
self.queue_out = output_queue
self.N = N
def run(self):
while True:
n = self.queue_out.qsize()
sys.stdout.write('\rdownloaded: %3.0f %%' % (100*n/self.N))
sys.stdout.flush()
#grabs host from queue
host,shot = self.queue.get()
d= DataSource()
try:
f = d.open(host)
f = load(f)
self.queue_out.put((shot,f))
except IOError:
print host+' was not found'
finally:
self.queue.task_done()
def ParallelDownload( diagn ,shots,n_threats=5):
url_0 = get_address(diagn)
start = time.time()
links = [ url_0 % s for s in shots]
queue = Queue.Queue()
output_queue = Queue.Queue()
if n_threats > 10:
print 'DDos attack :D, n_threats decreased to 10'
n_threats = 10
#spawn a pool of threads, and pass them queue instance
for i in range(n_threats):
t = ThreadUrl(queue,output_queue,len(links))
t.setDaemon(True)
t.start()
#populate queue with data
for host,shot in zip(links,shots):
queue.put((host,shot))
#wait on the queue until everything has been processed
queue.join()
print "\nmean time: %2.3fs" % ((time.time() - start)/len(shots))
results = {}
while not output_queue.empty():
shot,file = output_queue.get()
results[shot] = file
return results
def get_address(diagnostics):
address = DataSource().open('http://golem.fjfi.cvut.cz/utils/data/0/'+diagnostics+'.link' ).read()
ShotNo = int(DataSource().open('http://golem.fjfi.cvut.cz/shots/0/ShotNo').read())
address = re.sub( r'/'+str(ShotNo)+'/', r'/%d/', address)
return address
main()
