#from boxbranding import getMachineBrand, getMachineName import sys,os, os.path, re import urlparse, requests from twisted.web import client from twisted.internet import reactor, defer, ssl USER_AGENT = "Enigma2 HbbTV/1.1.1 (+PVR+RTSP+DL;OpenATV;;;)" ##################################################################################################### class HTTPProgressDownloader(client.HTTPDownloader): def __init__(self, url, outfile, headers=None): agent = USER_AGENT if headers and "user-agent" in headers: agent = headers["user-agent"] if headers and "User-Agent" in headers: agent = headers["User-Agent"] client.HTTPDownloader.__init__(self, url, outfile, headers=headers, agent=agent) self.status = None self.progress_callback = None self.deferred = defer.Deferred() def noPage(self, reason): if self.status == "304": print reason.getErrorMessage() client.HTTPDownloader.page(self, "") else: client.HTTPDownloader.noPage(self, reason) def gotHeaders(self, headers): if self.status == "200": if headers.has_key("content-length"): self.totalbytes = int(headers["content-length"][0]) else: self.totalbytes = 0 self.currentbytes = 0.0 return client.HTTPDownloader.gotHeaders(self, headers) def pagePart(self, packet): if self.status == "200": self.currentbytes += len(packet) if self.totalbytes and self.progress_callback: self.progress_callback(self.currentbytes, self.totalbytes) return client.HTTPDownloader.pagePart(self, packet) def pageEnd(self): return client.HTTPDownloader.pageEnd(self) class DownloadWithProgress: def __init__(self, url, outputfile, headers=None, limit=0, contextFactory=None, *args, **kwargs): self.limit = limit uri = urlparse.urlparse(url) scheme = uri.scheme host = uri.hostname port = uri.port if uri.port else 80 path = uri.path if not headers: headers = {"user-agent":USER_AGENT} self.factory = HTTPProgressDownloader(url, outputfile, headers, *args, **kwargs) if scheme == "https": self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory()) else: self.connection = reactor.connectTCP(host, port, self.factory) def start(self): return self.factory.deferred def stop(self): if self.connection: print "[stop]" self.connection.disconnect() def addProgress(self, progress_callback): print "[addProgress]" self.factory.progress_callback = progress_callback ##################################################################################################### class DownloadWithProgressFragmented: def __init__(self, url, outputfile, headers = None, limit = 0, contextFactory=None, *args, **kwargs): self.url = url self.outputfile = outputfile self.base_url = "/".join(url.split("/")[:-1])+"/" self.headers = headers if headers else {"user-agent":"Enigma2"} self.limit = limit self.agent = kwargs["agent"] if "agent" in kwargs else None self.cookie = kwargs["cookie"] if "cookie" in kwargs else None self.deferred = defer.Deferred() #self.deferred.addCallback(self.start_download) def start_download(self): print "Start download" try: r = requests.get(self.url,headers=self.headers) except Exception as e: #self.deferred.errback("Cannot open manifsest file - %s"%url) self.deferred.errback(e) if not r.content.startswith("#EXTM3U"): self.deferred.errback(Exception("Not valid manifest file - %s"%self.url)) streams = re.findall(r"#EXT-X-STREAM-INF:.*?BANDWIDTH=(\d+).*?\n(.+?)$", r.content, re.IGNORECASE | re.MULTILINE) if streams: sorted(streams, key=lambda item: int(item[0]), reverse=True) url = streams[0][1] if not url.startswith("http"): url = self.base_url + url try: r = requests.get(url, headers=self.headers) except Exception as e: self.deferred.errback(Exception("Cannot open manifsest file - %s"%url)) self.ts_list = re.findall(r"#EXTINF:([\d\.]+),.*?\n(.+?)$", r.content, re.IGNORECASE | re.MULTILINE) if not len(self.ts_list): self.deferred.errback(Exception("Cannot read fragment list in manifsest file - %s"%url)) self.ts_num = 0 self.type = "vod" if "#EXT-X-ENDLIST" in r.content else "live" self.currentbytes = 0.0 self.totalbytes = -1 self.currenttime = 0.0 self.totaltime = sum(map(float,zip(*self.ts_list)[0])) self.ts_file = open(self.outputfile, "wb") self.download_fragment() def download_fragment(self): if self.ts_num>=len(self.ts_list): pass print "Call later" reactor.callLater(10,self.update_manifest) reactor.callLater(10, self.download_fragment) else: print "Start fragment download" url = self.ts_list[self.ts_num][1] if not "://" in url: url = self.base_url+url self.d = client.getPage(url,headers = self.headers) self.d.addCallbacks(self.download_ok,self.download_err) def download_ok(self,content): content_length = len(content) self.currentbytes += content_length self.currenttime += float(self.ts_list[self.ts_num][0]) self.totalbytes = self.currentbytes * self.totaltime / self.currenttime self.ts_num += 1 #print "Fragment %s downloaded (%s)"%(self.ts_num,len(content)) self.ts_file.write(content) self.progress_callback(self.currentbytes, self.totalbytes) if self.type == "vod": if self.ts_num >= len(self.ts_list) or (self.limit and self.currenttime>self.limit): self.ts_file.close() self.download_finished() else: self.download_fragment() else: if self.limit and self.currenttime>self.limit: # TODO self.ts_file.close() self.download_finished() else: self.download_fragment() def update_manifest(self): self.d2 = client.getPage(self.url, headers=self.headers) self.d2.addCallbacks(self.update_manifest_ok, self.update_manifest_err) def update_manifest_ok(self,content): print "Update manifest" ts_list = re.findall(r"#EXTINF:([\d\.]+),\n(.+?)$", content, re.IGNORECASE | re.MULTILINE) last_ts = self.ts_list[-1] found = False for ts in ts_list: if ts == last_ts: found = True elif found: print "Append %s"%ts[1] self.ts_list.append(ts) #reactor.callLater(5,self.download_fragment) def update_manifest_err(self,content): return def download_err(self,content): self.deferred.errback("Error while downloading %s"%self.ts_list[self.ts_num][1]) def download_finished(self): self.totalbytes = self.currentbytes self.deferred.callback("Done") def start(self): reactor.callLater(1,self.start_download) return self.deferred def stop(self): self.deferred.errback() # TODO def addProgress(self, progress_callback): print "[addProgress]" self.progress_callback = progress_callback ##################################################################################################### def get_header(url,headers=None): headers = {"user-agent":USER_AGENT} r = requests.head(url,headers=headers) return r.headers def get_ext(mtype): stype = "http" if mtype in ("vnd.apple.mpegURL","application/x-mpegURL",'application/x-mpegurl',"application/vnd.apple.mpegurl"): return ".ts","hls" elif mtype in ("application/dash+xml"): return ".ts","dash" # TODO dash stream type could be different ! elif mtype in ("video/mp4"): return ".mp4","http" elif mtype in ("video/MP2T","video/mp2t"): return ".ts","http" elif mtype in ("video/x-flv"): return ".flv","http" elif mtype in ("video/quicktime"): return ".mov","http" elif mtype in ("video/x-msvideo"): return ".avi","http" elif mtype in ("video/x-ms-wmv"): return ".wmv","http" elif mtype in ("video/x-matroska"): return ".mkv","http" else: return ".mp4","http" ############################################## def print_progress(currentbytes, totalbytes): progress = float(currentbytes)/float(totalbytes)*100 print "%s (%i/%i)"%(progress,currentbytes,totalbytes) def download_ok(*args): print "Download OK" reactor.stop() def download_err(e): print "Download Error %s"%e.getBriefTraceback() pass def stop(): reactor.stop() ############################################### def download_vide(stream): stream = stream[0] url = stream["url"] headers = stream["headers"] output = stream["name"].replace("\\"," ").replace(":"," ").replace("|"," ") try: h = get_header(url,headers={"user-agent":"Enigma2"}) mtype = h.get("content-type") ext,stream_type = get_ext(mtype) except: ext,stream_type = (".ts","hls") #output = urlparse.urlparse(url)[2].split('/')[-1] + ext output = output+ext output = os.path.join("downloads", output) if stream_type == "hls": d = DownloadWithProgressFragmented(url,output,headers={"user-agent":"Enigma2"}) else: d = DownloadWithProgress(url,output,headers={"user-agent":"Enigma2"}) d.addProgress(print_progress) d.start().addCallback(download_ok).addErrback(download_err) reactor.run() if __name__ == "__main__": if len(sys.argv)>2: url= sys.argv[1] output = sys.argv[1] else: url = "http://walterebert.com/playground/video/hls/ts/480x270.m3u8" url = "https://r3---sn-bavc5ajvh-gpme.googlevideo.com/videoplayback?key=yt6&mime=video%2Fmp4&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cupn%2Cexpire&expire=1490986184&lmt=1490940183963773&dur=1302.639&itag=18&ratebypass=yes&mm=31&requiressl=yes&ipbits=0&upn=azFGj8gY02w&ip=85.254.87.15&pl=23&ei=aFDeWLzqDcn-dLC_gdAM&signature=083F353AC09CD98A70AD7D9438DD3C91C781166B.715456B9C35F040BDC4728CA76A0D1779B684A90&source=youtube&mv=m&mt=1490964451&ms=au&mn=sn-bavc5ajvh-gpme&gir=yes&clen=73596250&id=o-AGH9y-hWn1MtW1VzCyI_8XYYEWODsTDBZbfagQH3BrfQ&initcwndbps=4493750" #url = "http://techslides.com/demos/sample-videos/small.mp4" #url = "http://wx17.poiuytrew.pw/s/c507282042b1bf25e0b72c34a68426f3/hd_30/Jackie.2016.D.iTunes.BDRip.1080p_720.mp4" #url = "http://player.tvnet.lv/live/amlst:11/chunklist_w361981294_b528000.m3u8" #url = "http://vod-hls-uk-live.akamaized.net/usp/auth/vod/piff_abr_full_hd/a3e90e-b08ktytr/vf_b08ktytr_f9d55583-afc7-49bb-9bf4-d8f1ac99f56f.ism.hlsv2.ism/vf_b08ktytr_f9d55583-afc7-49bb-9bf4-d8f1ac99f56f.ism.hlsv2-audio=128000-video=5070000.m3u8" #url = "https://58174450afee9.streamlock.net/vod/mp4:_definst_/f/e/8e49fc32.mp4/playlist.m3u8?safwerwfasendtime=1490877870&safwerwfasstarttime=1490859339&safwerwfashash=hS2FfVZysQVazBQ6RJn1IhUevBkKxIF09Ly3BjfT43U=" try: h = get_header(url,headers={"user-agent":"Enigma2"}) mtype = h.get("content-type") ext,stream_type = get_ext(mtype) except: ext,stream_type = (".ts","hls") output = urlparse.urlparse(url)[2].split('/')[-1] + ext output = os.path.join("downloads", output) if stream_type == "hls": d = DownloadWithProgressFragmented(url,output,headers={"user-agent":"Enigma2"}) else: d = DownloadWithProgress(url,output,headers={"user-agent":"Enigma2"}) d.addProgress(print_progress) d.start().addCallback(download_ok).addErrback(download_err) reactor.run()