content/Downloader.py

#from boxbranding import getMachineBrand, getMachineName
import sys,os, os.path, re
import urlparse, requests
from twisted.web import client
from twisted.internet import reactor, defer, ssl

USER_AGENT = "Enigma2 HbbTV/1.1.1 (+PVR+RTSP+DL;OpenATV;;;)"

#####################################################################################################
class HTTPProgressDownloader(client.HTTPDownloader):
    def __init__(self, url, outfile, headers=None):
        agent = USER_AGENT
        if headers and "user-agent" in headers:
            agent = headers["user-agent"]
        if headers and "User-Agent" in headers:
            agent = headers["User-Agent"]
        client.HTTPDownloader.__init__(self, url, outfile, headers=headers, agent=agent)
        self.status = None
        self.progress_callback = None
        self.deferred = defer.Deferred()

    def noPage(self, reason):
        if self.status == "304":
            print reason.getErrorMessage()
            client.HTTPDownloader.page(self, "")
        else:
            client.HTTPDownloader.noPage(self, reason)

    def gotHeaders(self, headers):
        if self.status == "200":
            if headers.has_key("content-length"):
                self.totalbytes = int(headers["content-length"][0])
            else:
                self.totalbytes = 0
            self.currentbytes = 0.0
        return client.HTTPDownloader.gotHeaders(self, headers)

    def pagePart(self, packet):
        if self.status == "200":
            self.currentbytes += len(packet)
        if self.totalbytes and self.progress_callback:
            self.progress_callback(self.currentbytes, self.totalbytes)
        return client.HTTPDownloader.pagePart(self, packet)

    def pageEnd(self):
        return client.HTTPDownloader.pageEnd(self)

class DownloadWithProgress:
    def __init__(self, url, outputfile, headers=None, limit=0, contextFactory=None, *args, **kwargs):
        self.limit = limit
        uri = urlparse.urlparse(url)
        scheme = uri.scheme
        host = uri.hostname
        port = uri.port if uri.port else 80
        path = uri.path
        if not headers:
            headers = {"user-agent":USER_AGENT}
        self.factory = HTTPProgressDownloader(url, outputfile, headers, *args, **kwargs)
        if scheme == "https":
            self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory())
        else:
            self.connection = reactor.connectTCP(host, port, self.factory)

    def start(self):
        return self.factory.deferred

    def stop(self):
        if self.connection:
            print "[stop]"
            self.connection.disconnect()

    def addProgress(self, progress_callback):
        print "[addProgress]"
        self.factory.progress_callback = progress_callback

#####################################################################################################
class DownloadWithProgressFragmented:
    def __init__(self, url, outputfile, headers = None, limit = 0, contextFactory=None, *args, **kwargs):
        self.url = url
        self.outputfile = outputfile
        self.base_url = "/".join(url.split("/")[:-1])+"/"
        self.headers = headers if headers else  {"user-agent":"Enigma2"}
        self.limit = limit
        self.agent = kwargs["agent"] if "agent" in kwargs else None
        self.cookie = kwargs["cookie"] if "cookie" in kwargs else None
        self.deferred = defer.Deferred()
        #self.deferred.addCallback(self.start_download)

    def start_download(self):
        print "Start download"
        try:
            r = requests.get(self.url,headers=self.headers)
        except Exception as e:
            #self.deferred.errback("Cannot open manifsest file - %s"%url)
            self.deferred.errback(e)
        if not r.content.startswith("#EXTM3U"):
            self.deferred.errback(Exception("Not valid manifest file - %s"%self.url))
        streams = re.findall(r"#EXT-X-STREAM-INF:.*?BANDWIDTH=(\d+).*?\n(.+?)$", r.content, re.IGNORECASE | re.MULTILINE)
        if streams:
            sorted(streams, key=lambda item: int(item[0]), reverse=True)
            url = streams[0][1]
            if not url.startswith("http"):
                url = self.base_url + url
            try:
                r = requests.get(url, headers=self.headers)
            except Exception as e:
                self.deferred.errback(Exception("Cannot open manifsest file - %s"%url))
        self.ts_list = re.findall(r"#EXTINF:([\d\.]+),.*?\n(.+?)$", r.content, re.IGNORECASE | re.MULTILINE)
        if not len(self.ts_list):
            self.deferred.errback(Exception("Cannot read fragment list in  manifsest file - %s"%url))
        self.ts_num = 0
        self.type = "vod" if "#EXT-X-ENDLIST" in r.content else "live"
        self.currentbytes = 0.0
        self.totalbytes = -1
        self.currenttime = 0.0
        self.totaltime = sum(map(float,zip(*self.ts_list)[0]))
        self.ts_file = open(self.outputfile, "wb")
        self.download_fragment()

    def download_fragment(self):
        if self.ts_num>=len(self.ts_list):
            pass
            #print "Call later"
            reactor.callLater(10,self.update_manifest)
            reactor.callLater(10, self.download_fragment)
        else:
            print "Start fragment download"
            url = self.ts_list[self.ts_num][1]
            if not "://" in url:
                url = self.base_url+url
            self.d = client.getPage(url,headers = self.headers)
            self.d.addCallbacks(self.download_ok,self.download_err)


    def download_ok(self,content):
        content_length = len(content)
        self.currentbytes += content_length
        self.currenttime += float(self.ts_list[self.ts_num][0])
        self.totalbytes = self.currentbytes * self.totaltime / self.currenttime
        self.ts_num += 1
        #print "Fragment %s downloaded (%s)"%(self.ts_num,len(content))
        self.ts_file.write(content)
        self.progress_callback(self.currentbytes, self.totalbytes)
        if self.type == "vod":
            if self.ts_num >= len(self.ts_list) or (self.limit and self.currenttime>self.limit):
                self.ts_file.close()
                self.download_finished()
            else:
                self.download_fragment()
        else:
            if self.limit and self.currenttime>self.limit: # TODO
                self.ts_file.close()
                self.download_finished()
            else:
                self.download_fragment()

    def update_manifest(self):
        self.d2 = client.getPage(self.url, headers=self.headers)
        self.d2.addCallbacks(self.update_manifest_ok, self.update_manifest_err)

    def update_manifest_ok(self,content):
        print "Update manifest"
        ts_list = re.findall(r"#EXTINF:([\d\.]+),\n(.+?)$", content, re.IGNORECASE | re.MULTILINE)
        last_ts = self.ts_list[-1]
        found = False
        for ts in ts_list:
            if ts == last_ts:
                found = True
            elif found:
                print "Append %s"%ts[1]
                self.ts_list.append(ts)
        #reactor.callLater(5,self.download_fragment)

    def update_manifest_err(self,content):
        return

    def download_err(self,content):
        self.deferred.errback("Error while downloading %s"%self.ts_list[self.ts_num][1])

    def download_finished(self):
        self.totalbytes = self.currentbytes
        self.deferred.callback("Done")

    def start(self):
        reactor.callLater(1,self.start_download)
        return self.deferred

    def stop(self):
        self.deferred.errback() # TODO

    def addProgress(self, progress_callback):
        print "[addProgress]"
        self.progress_callback = progress_callback

#####################################################################################################
def get_header(url,headers=None):
    headers = {"user-agent":USER_AGENT}
    r = requests.head(url,headers=headers)
    return r.headers

def get_ext(mtype):
    stype = "http"
    if mtype in ("vnd.apple.mpegURL","application/x-mpegURL",'application/x-mpegurl',"application/vnd.apple.mpegurl"):
        return ".ts","hls"
    elif mtype in ("application/dash+xml"):
        return ".ts","dash" # TODO dash stream type  could be different !
    elif mtype in ("video/mp4"):
        return ".mp4","http"
    elif mtype in ("video/MP2T","video/mp2t"):
        return ".ts","http"
    elif mtype in ("video/x-flv"):
        return ".flv","http"
    elif mtype in ("video/quicktime"):
        return ".mov","http"
    elif mtype in ("video/x-msvideo"):
        return ".avi","http"
    elif mtype in ("video/x-ms-wmv"):
        return ".wmv","http"
    elif mtype in ("video/x-matroska"):
        return ".mkv","http"
    else:
        return ".mp4","http"



##############################################
def print_progress(currentbytes, totalbytes):
    progress = float(currentbytes)/float(totalbytes)*100
    print "%s (%i/%i)"%(progress,currentbytes,totalbytes)

def download_ok(*args):
    print "Download OK"
    reactor.stop()

def download_err(e):
    print "Download Error %s"%e.getBriefTraceback()
    pass
def stop():
    reactor.stop()
###############################################

def download_video(stream):
    stream = stream[0]
    url = stream["url"]
    headers = stream["headers"]
    output = stream["name"].replace("\\"," ").replace(":"," ").replace("|"," ")
    try:
        h = get_header(url,headers={"user-agent":"Enigma2"})
        mtype = h.get("content-type")
        ext,stream_type = get_ext(mtype)
    except:
        ext,stream_type = (".ts","hls")
    #output = urlparse.urlparse(url)[2].split('/')[-1] + ext
    output = output+ext
    output = os.path.join("downloads", output)
    if stream_type == "hls":
        d = DownloadWithProgressFragmented(url,output,headers={"user-agent":"Enigma2"})
    else:
        d = DownloadWithProgress(url,output,headers={"user-agent":"Enigma2"})
    d.addProgress(print_progress)
    d.start().addCallback(download_ok).addErrback(download_err)
    reactor.run()


if __name__ == "__main__":
    if len(sys.argv)>2:
        url= sys.argv[1]
        output = sys.argv[1]
    else:
        url = "http://walterebert.com/playground/video/hls/ts/480x270.m3u8"
        url = "https://r3---sn-bavc5ajvh-gpme.googlevideo.com/videoplayback?key=yt6&mime=video%2Fmp4&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cupn%2Cexpire&expire=1490986184&lmt=1490940183963773&dur=1302.639&itag=18&ratebypass=yes&mm=31&requiressl=yes&ipbits=0&upn=azFGj8gY02w&ip=85.254.87.15&pl=23&ei=aFDeWLzqDcn-dLC_gdAM&signature=083F353AC09CD98A70AD7D9438DD3C91C781166B.715456B9C35F040BDC4728CA76A0D1779B684A90&source=youtube&mv=m&mt=1490964451&ms=au&mn=sn-bavc5ajvh-gpme&gir=yes&clen=73596250&id=o-AGH9y-hWn1MtW1VzCyI_8XYYEWODsTDBZbfagQH3BrfQ&initcwndbps=4493750"
        #url = "http://techslides.com/demos/sample-videos/small.mp4"
        #url = "http://wx17.poiuytrew.pw/s/c507282042b1bf25e0b72c34a68426f3/hd_30/Jackie.2016.D.iTunes.BDRip.1080p_720.mp4"
        #url = "http://player.tvnet.lv/live/amlst:11/chunklist_w361981294_b528000.m3u8"
        #url = "http://vod-hls-uk-live.akamaized.net/usp/auth/vod/piff_abr_full_hd/a3e90e-b08ktytr/vf_b08ktytr_f9d55583-afc7-49bb-9bf4-d8f1ac99f56f.ism.hlsv2.ism/vf_b08ktytr_f9d55583-afc7-49bb-9bf4-d8f1ac99f56f.ism.hlsv2-audio=128000-video=5070000.m3u8"
        #url = "https://58174450afee9.streamlock.net/vod/mp4:_definst_/f/e/8e49fc32.mp4/playlist.m3u8?safwerwfasendtime=1490877870&safwerwfasstarttime=1490859339&safwerwfashash=hS2FfVZysQVazBQ6RJn1IhUevBkKxIF09Ly3BjfT43U="
    try:
        h = get_header(url,headers={"user-agent":"Enigma2"})
        mtype = h.get("content-type")
        ext,stream_type = get_ext(mtype)
    except:
        ext,stream_type = (".ts","hls")
    output = urlparse.urlparse(url)[2].split('/')[-1] + ext
    output = os.path.join("downloads", output)
    if stream_type == "hls":
        d = DownloadWithProgressFragmented(url,output,headers={"user-agent":"Enigma2"})
    else:
        d = DownloadWithProgress(url,output,headers={"user-agent":"Enigma2"})
    d.addProgress(print_progress)
    d.start().addCallback(download_ok).addErrback(download_err)
    reactor.run()