work

ContentSources.py

@@ -165,15 +165,11 @@ class ContentSources(object):
         else:
             return None
 
-if __name__ == "__main__":
 
-    show_hidden = False
-    sources = ContentSources("sources")
-    if len(sys.argv)>1:
-        data= sys.argv[1]
-    else:
-        data = "config::home"
 
+def run(data="config::home"):
+
+    sources = ContentSources("sources")
     #options = sources.options_read("ltc")
     #print options
     history = []
@@ -259,4 +255,10 @@ if __name__ == "__main__":
             traceback.print_exc()
             raw_input("Continue?")
 
-
+if __name__ == "__main__":
+    show_hidden = False
+    if len(sys.argv)>1:
+        data= sys.argv[1]
+    else:
+        data = "config::home"
+    run(data)

Downloader.py

@@ -0,0 +1,293 @@
+#from boxbranding import getMachineBrand, getMachineName
+import sys,os, os.path, re
+import urlparse, requests
+from twisted.web import client
+from twisted.internet import reactor, defer, ssl
+
+USER_AGENT = "Enigma2 HbbTV/1.1.1 (+PVR+RTSP+DL;OpenATV;;;)"
+
+#####################################################################################################
+class HTTPProgressDownloader(client.HTTPDownloader):
+    def __init__(self, url, outfile, headers=None):
+        agent = USER_AGENT
+        if headers and "user-agent" in headers:
+            agent = headers["user-agent"]
+        if headers and "User-Agent" in headers:
+            agent = headers["User-Agent"]
+        client.HTTPDownloader.__init__(self, url, outfile, headers=headers, agent=agent)
+        self.status = None
+        self.progress_callback = None
+        self.deferred = defer.Deferred()
+
+    def noPage(self, reason):
+        if self.status == "304":
+            print reason.getErrorMessage()
+            client.HTTPDownloader.page(self, "")
+        else:
+            client.HTTPDownloader.noPage(self, reason)
+
+    def gotHeaders(self, headers):
+        if self.status == "200":
+            if headers.has_key("content-length"):
+                self.totalbytes = int(headers["content-length"][0])
+            else:
+                self.totalbytes = 0
+            self.currentbytes = 0.0
+        return client.HTTPDownloader.gotHeaders(self, headers)
+
+    def pagePart(self, packet):
+        if self.status == "200":
+            self.currentbytes += len(packet)
+        if self.totalbytes and self.progress_callback:
+            self.progress_callback(self.currentbytes, self.totalbytes)
+        return client.HTTPDownloader.pagePart(self, packet)
+
+    def pageEnd(self):
+        return client.HTTPDownloader.pageEnd(self)
+
+class DownloadWithProgress:
+    def __init__(self, url, outputfile, headers=None, limit=0, contextFactory=None, *args, **kwargs):
+        self.limit = limit
+        uri = urlparse.urlparse(url)
+        scheme = uri.scheme
+        host = uri.hostname
+        port = uri.port if uri.port else 80
+        path = uri.path
+        if not headers:
+            headers = {"user-agent":USER_AGENT}
+        self.factory = HTTPProgressDownloader(url, outputfile, headers, *args, **kwargs)
+        if scheme == "https":
+            self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory())
+        else:
+            self.connection = reactor.connectTCP(host, port, self.factory)
+
+    def start(self):
+        return self.factory.deferred
+
+    def stop(self):
+        if self.connection:
+            print "[stop]"
+            self.connection.disconnect()
+
+    def addProgress(self, progress_callback):
+        print "[addProgress]"
+        self.factory.progress_callback = progress_callback
+
+#####################################################################################################
+class DownloadWithProgressFragmented:
+    def __init__(self, url, outputfile, headers = None, limit = 0, contextFactory=None, *args, **kwargs):
+        self.url = url
+        self.outputfile = outputfile
+        self.base_url = "/".join(url.split("/")[:-1])+"/"
+        self.headers = headers if headers else  {"user-agent":"Enigma2"}
+        self.limit = limit
+        self.agent = kwargs["agent"] if "agent" in kwargs else None
+        self.cookie = kwargs["cookie"] if "cookie" in kwargs else None
+        self.deferred = defer.Deferred()
+        #self.deferred.addCallback(self.start_download)
+
+    def start_download(self):
+        print "Start download"
+        try:
+            r = requests.get(self.url,headers=self.headers)
+        except Exception as e:
+            #self.deferred.errback("Cannot open manifsest file - %s"%url)
+            self.deferred.errback(e)
+        if not r.content.startswith("#EXTM3U"):
+            self.deferred.errback(Exception("Not valid manifest file - %s"%self.url))
+        streams = re.findall(r"#EXT-X-STREAM-INF:.*?BANDWIDTH=(\d+).*?\n(.+?)$", r.content, re.IGNORECASE | re.MULTILINE)
+        if streams:
+            sorted(streams, key=lambda item: int(item[0]), reverse=True)
+            url = streams[0][1]
+            if not url.startswith("http"):
+                url = self.base_url + url
+            try:
+                r = requests.get(url, headers=self.headers)
+            except Exception as e:
+                self.deferred.errback(Exception("Cannot open manifsest file - %s"%url))
+        self.ts_list = re.findall(r"#EXTINF:([\d\.]+),.*?\n(.+?)$", r.content, re.IGNORECASE | re.MULTILINE)
+        if not len(self.ts_list):
+            self.deferred.errback(Exception("Cannot read fragment list in  manifsest file - %s"%url))
+        self.ts_num = 0
+        self.type = "vod" if "#EXT-X-ENDLIST" in r.content else "live"
+        self.currentbytes = 0.0
+        self.totalbytes = -1
+        self.currenttime = 0.0
+        self.totaltime = sum(map(float,zip(*self.ts_list)[0]))
+        self.ts_file = open(self.outputfile, "wb")
+        self.download_fragment()
+
+    def download_fragment(self):
+        if self.ts_num>=len(self.ts_list):
+            pass
+            #print "Call later"
+            reactor.callLater(10,self.update_manifest)
+            reactor.callLater(10, self.download_fragment)
+        else:
+            print "Start fragment download"
+            url = self.ts_list[self.ts_num][1]
+            if not "://" in url:
+                url = self.base_url+url
+            self.d = client.getPage(url,headers = self.headers)
+            self.d.addCallbacks(self.download_ok,self.download_err)
+
+
+    def download_ok(self,content):
+        content_length = len(content)
+        self.currentbytes += content_length
+        self.currenttime += float(self.ts_list[self.ts_num][0])
+        self.totalbytes = self.currentbytes * self.totaltime / self.currenttime
+        self.ts_num += 1
+        #print "Fragment %s downloaded (%s)"%(self.ts_num,len(content))
+        self.ts_file.write(content)
+        self.progress_callback(self.currentbytes, self.totalbytes)
+        if self.type == "vod":
+            if self.ts_num >= len(self.ts_list) or (self.limit and self.currenttime>self.limit):
+                self.ts_file.close()
+                self.download_finished()
+            else:
+                self.download_fragment()
+        else:
+            if self.limit and self.currenttime>self.limit: # TODO
+                self.ts_file.close()
+                self.download_finished()
+            else:
+                self.download_fragment()
+
+    def update_manifest(self):
+        self.d2 = client.getPage(self.url, headers=self.headers)
+        self.d2.addCallbacks(self.update_manifest_ok, self.update_manifest_err)
+
+    def update_manifest_ok(self,content):
+        print "Update manifest"
+        ts_list = re.findall(r"#EXTINF:([\d\.]+),\n(.+?)$", content, re.IGNORECASE | re.MULTILINE)
+        last_ts = self.ts_list[-1]
+        found = False
+        for ts in ts_list:
+            if ts == last_ts:
+                found = True
+            elif found:
+                print "Append %s"%ts[1]
+                self.ts_list.append(ts)
+        #reactor.callLater(5,self.download_fragment)
+
+    def update_manifest_err(self,content):
+        return
+
+    def download_err(self,content):
+        self.deferred.errback("Error while downloading %s"%self.ts_list[self.ts_num][1])
+
+    def download_finished(self):
+        self.totalbytes = self.currentbytes
+        self.deferred.callback("Done")
+
+    def start(self):
+        reactor.callLater(1,self.start_download)
+        return self.deferred
+
+    def stop(self):
+        self.deferred.errback() # TODO
+
+    def addProgress(self, progress_callback):
+        print "[addProgress]"
+        self.progress_callback = progress_callback
+
+#####################################################################################################
+def get_header(url,headers=None):
+    headers = {"user-agent":USER_AGENT}
+    r = requests.head(url,headers=headers)
+    return r.headers
+
+def get_ext(mtype):
+    stype = "http"
+    if mtype in ("vnd.apple.mpegURL","application/x-mpegURL",'application/x-mpegurl',"application/vnd.apple.mpegurl"):
+        return ".ts","hls"
+    elif mtype in ("application/dash+xml"):
+        return ".ts","dash" # TODO dash stream type  could be different !
+    elif mtype in ("video/mp4"):
+        return ".mp4","http"
+    elif mtype in ("video/MP2T","video/mp2t"):
+        return ".ts","http"
+    elif mtype in ("video/x-flv"):
+        return ".flv","http"
+    elif mtype in ("video/quicktime"):
+        return ".mov","http"
+    elif mtype in ("video/x-msvideo"):
+        return ".avi","http"
+    elif mtype in ("video/x-ms-wmv"):
+        return ".wmv","http"
+    elif mtype in ("video/x-matroska"):
+        return ".mkv","http"
+    else:
+        return ".mp4","http"
+
+
+
+##############################################
+def print_progress(currentbytes, totalbytes):
+    progress = float(currentbytes)/float(totalbytes)*100
+    print "%s (%i/%i)"%(progress,currentbytes,totalbytes)
+
+def download_ok(*args):
+    print "Download OK"
+    reactor.stop()
+
+def download_err(e):
+    print "Download Error %s"%e.getBriefTraceback()
+    pass
+def stop():
+    reactor.stop()
+###############################################
+
+def download_video(stream):
+    stream = stream[0]
+    url = stream["url"]
+    headers = stream["headers"]
+    output = stream["name"].replace("\\"," ").replace(":"," ").replace("|"," ")
+    try:
+        h = get_header(url,headers={"user-agent":"Enigma2"})
+        mtype = h.get("content-type")
+        ext,stream_type = get_ext(mtype)
+    except:
+        ext,stream_type = (".ts","hls")
+    #output = urlparse.urlparse(url)[2].split('/')[-1] + ext
+    output = output+ext
+    output = os.path.join("downloads", output)
+    if stream_type == "hls":
+        d = DownloadWithProgressFragmented(url,output,headers={"user-agent":"Enigma2"})
+    else:
+        d = DownloadWithProgress(url,output,headers={"user-agent":"Enigma2"})
+    d.addProgress(print_progress)
+    d.start().addCallback(download_ok).addErrback(download_err)
+    reactor.run()
+
+
+if __name__ == "__main__":
+    if len(sys.argv)>2:
+        url= sys.argv[1]
+        output = sys.argv[1]
+    else:
+        url = "http://walterebert.com/playground/video/hls/ts/480x270.m3u8"
+        url = "https://r3---sn-bavc5ajvh-gpme.googlevideo.com/videoplayback?key=yt6&mime=video%2Fmp4&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cupn%2Cexpire&expire=1490986184&lmt=1490940183963773&dur=1302.639&itag=18&ratebypass=yes&mm=31&requiressl=yes&ipbits=0&upn=azFGj8gY02w&ip=85.254.87.15&pl=23&ei=aFDeWLzqDcn-dLC_gdAM&signature=083F353AC09CD98A70AD7D9438DD3C91C781166B.715456B9C35F040BDC4728CA76A0D1779B684A90&source=youtube&mv=m&mt=1490964451&ms=au&mn=sn-bavc5ajvh-gpme&gir=yes&clen=73596250&id=o-AGH9y-hWn1MtW1VzCyI_8XYYEWODsTDBZbfagQH3BrfQ&initcwndbps=4493750"
+        #url = "http://techslides.com/demos/sample-videos/small.mp4"
+        #url = "http://wx17.poiuytrew.pw/s/c507282042b1bf25e0b72c34a68426f3/hd_30/Jackie.2016.D.iTunes.BDRip.1080p_720.mp4"
+        #url = "http://player.tvnet.lv/live/amlst:11/chunklist_w361981294_b528000.m3u8"
+        #url = "http://vod-hls-uk-live.akamaized.net/usp/auth/vod/piff_abr_full_hd/a3e90e-b08ktytr/vf_b08ktytr_f9d55583-afc7-49bb-9bf4-d8f1ac99f56f.ism.hlsv2.ism/vf_b08ktytr_f9d55583-afc7-49bb-9bf4-d8f1ac99f56f.ism.hlsv2-audio=128000-video=5070000.m3u8"
+        #url = "https://58174450afee9.streamlock.net/vod/mp4:_definst_/f/e/8e49fc32.mp4/playlist.m3u8?safwerwfasendtime=1490877870&safwerwfasstarttime=1490859339&safwerwfashash=hS2FfVZysQVazBQ6RJn1IhUevBkKxIF09Ly3BjfT43U="
+    try:
+        h = get_header(url,headers={"user-agent":"Enigma2"})
+        mtype = h.get("content-type")
+        ext,stream_type = get_ext(mtype)
+    except:
+        ext,stream_type = (".ts","hls")
+    output = urlparse.urlparse(url)[2].split('/')[-1] + ext
+    output = os.path.join("downloads", output)
+    if stream_type == "hls":
+        d = DownloadWithProgressFragmented(url,output,headers={"user-agent":"Enigma2"})
+    else:
+        d = DownloadWithProgress(url,output,headers={"user-agent":"Enigma2"})
+    d.addProgress(print_progress)
+    d.start().addCallback(download_ok).addErrback(download_err)
+    reactor.run()
+
+