0.1.35

download_service.py

+# -*- coding: utf-8 -*-
+import os,os.path,sys, urllib, traceback
+from kodiswift import Plugin, ListItem, storage
+from kodiswift import xbmc, xbmcgui, xbmcplugin, xbmcvfs, CLI_MODE
+#from resources.lib import ContentSources, util
+#sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)),"resources","lib","sources"))
+import playstreamproxy
+#monitor = xbmc.Monitor()
+
+
+plugin = Plugin()
+plugin.load_addon_settings()
+port = plugin.get_setting("general_port",int)
+start_proxy = plugin.get_setting("general_proxy",bool)
+host = "localhost"
+if start_proxy:
+    plugin.notify("Starting playstreamproxy","Info",10000, xbmcgui.NOTIFICATION_INFO)
+    playstreamproxy.start(host, port)

resources/lib/Downloader.py

+#from boxbranding import getMachineBrand, getMachineName
+import sys,os, os.path, re
+import urlparse, requests
+from twisted.web import client
+from twisted.internet import reactor, defer, ssl
+
+USER_AGENT = "Enigma2 HbbTV/1.1.1 (+PVR+RTSP+DL;OpenATV;;;)"
+
+#####################################################################################################
+class HTTPProgressDownloader(client.HTTPDownloader):
+    def __init__(self, url, outfile, headers=None):
+        agent = USER_AGENT
+        if headers and "user-agent" in headers:
+            agent = headers["user-agent"]
+        if headers and "User-Agent" in headers:
+            agent = headers["User-Agent"]
+        client.HTTPDownloader.__init__(self, url, outfile, headers=headers, agent=agent)
+        self.status = None
+        self.progress_callback = None
+        self.deferred = defer.Deferred()
+
+    def noPage(self, reason):
+        if self.status == "304":
+            print reason.getErrorMessage()
+            client.HTTPDownloader.page(self, "")
+        else:
+            client.HTTPDownloader.noPage(self, reason)
+
+    def gotHeaders(self, headers):
+        if self.status == "200":
+            if headers.has_key("content-length"):
+                self.totalbytes = int(headers["content-length"][0])
+            else:
+                self.totalbytes = 0
+            self.currentbytes = 0.0
+        return client.HTTPDownloader.gotHeaders(self, headers)
+
+    def pagePart(self, packet):
+        if self.status == "200":
+            self.currentbytes += len(packet)
+        if self.totalbytes and self.progress_callback:
+            self.progress_callback(self.currentbytes, self.totalbytes)
+        return client.HTTPDownloader.pagePart(self, packet)
+
+    def pageEnd(self):
+        return client.HTTPDownloader.pageEnd(self)
+
+class DownloadWithProgress:
+    def __init__(self, url, outputfile, headers=None, limit=0, contextFactory=None, *args, **kwargs):
+        self.limit = limit
+        uri = urlparse.urlparse(url)
+        scheme = uri.scheme
+        host = uri.hostname
+        port = uri.port if uri.port else 80
+        path = uri.path
+        if not headers:
+            headers = {"user-agent":USER_AGENT}
+        self.factory = HTTPProgressDownloader(url, outputfile, headers, *args, **kwargs)
+        if scheme == "https":
+            self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory())
+        else:
+            self.connection = reactor.connectTCP(host, port, self.factory)
+
+    def start(self):
+        return self.factory.deferred
+
+    def stop(self):
+        if self.connection:
+            print "[stop]"
+            self.connection.disconnect()
+
+    def addProgress(self, progress_callback):
+        print "[addProgress]"
+        self.factory.progress_callback = progress_callback
+
+#####################################################################################################
+class DownloadWithProgressFragmented:
+    def __init__(self, url, outputfile, headers = None, limit = 0, contextFactory=None, *args, **kwargs):
+        self.url = url
+        self.outputfile = outputfile
+        self.base_url = "/".join(url.split("/")[:-1])+"/"
+        self.headers = headers if headers else  {"user-agent":"Enigma2"}
+        self.limit = limit
+        self.agent = kwargs["agent"] if "agent" in kwargs else None
+        self.cookie = kwargs["cookie"] if "cookie" in kwargs else None
+        self.deferred = defer.Deferred()
+        #self.deferred.addCallback(self.start_download)
+
+    def start_download(self):
+        print "Start download"
+        try:
+            r = requests.get(self.url,headers=self.headers)
+        except Exception as e:
+            #self.deferred.errback("Cannot open manifsest file - %s"%url)
+            self.deferred.errback(e)
+        if not r.content.startswith("#EXTM3U"):
+            self.deferred.errback(Exception("Not valid manifest file - %s"%self.url))
+        streams = re.findall(r"#EXT-X-STREAM-INF:.*?BANDWIDTH=(\d+).*?\n(.+?)$", r.content, re.IGNORECASE | re.MULTILINE)
+        if streams:
+            sorted(streams, key=lambda item: int(item[0]), reverse=True)
+            url = streams[0][1]
+            if not url.startswith("http"):
+                url = self.base_url + url
+            try:
+                r = requests.get(url, headers=self.headers)
+            except Exception as e:
+                self.deferred.errback(Exception("Cannot open manifsest file - %s"%url))
+        self.ts_list = re.findall(r"#EXTINF:([\d\.]+),.*?\n(.+?)$", r.content, re.IGNORECASE | re.MULTILINE)
+        if not len(self.ts_list):
+            self.deferred.errback(Exception("Cannot read fragment list in  manifsest file - %s"%url))
+        self.ts_num = 0
+        self.type = "vod" if "#EXT-X-ENDLIST" in r.content else "live"
+        self.currentbytes = 0.0
+        self.totalbytes = -1
+        self.currenttime = 0.0
+        self.totaltime = sum(map(float,zip(*self.ts_list)[0]))
+        try:
+            self.ts_file = open(self.outputfile, "wb")
+        except Exception as e:
+            self.deferred.errback(Exception("Cannot open output file - %s" % self.outputfile))
+        self.download_fragment()
+
+    def download_fragment(self):
+        if self.ts_num>=len(self.ts_list):
+            pass
+            print "Call later"
+            reactor.callLater(10,self.update_manifest)
+            reactor.callLater(10, self.download_fragment)
+        else:
+            print "Start fragment download"
+            url = self.ts_list[self.ts_num][1]
+            if not "://" in url:
+                url = self.base_url+url
+            self.d = client.getPage(url,headers = self.headers)
+            self.d.addCallbacks(self.download_ok,self.download_err)
+
+
+    def download_ok(self,content):
+        content_length = len(content)
+        self.currentbytes += content_length
+        self.currenttime += float(self.ts_list[self.ts_num][0])
+        self.totalbytes = self.currentbytes * self.totaltime / self.currenttime
+        self.ts_num += 1
+        #print "Fragment %s downloaded (%s)"%(self.ts_num,len(content))
+        self.ts_file.write(content)
+        self.progress_callback(self.currentbytes, self.totalbytes)
+        if self.type == "vod":
+            if self.ts_num >= len(self.ts_list) or (self.limit and self.currenttime>self.limit):
+                self.ts_file.close()
+                self.download_finished()
+            else:
+                self.download_fragment()
+        else:
+            if self.limit and self.currenttime>self.limit: # TODO
+                self.ts_file.close()
+                self.download_finished()
+            else:
+                self.download_fragment()
+
+    def update_manifest(self):
+        self.d2 = client.getPage(self.url, headers=self.headers)
+        self.d2.addCallbacks(self.update_manifest_ok, self.update_manifest_err)
+
+    def update_manifest_ok(self,content):
+        print "Update manifest"
+        ts_list = re.findall(r"#EXTINF:([\d\.]+),\n(.+?)$", content, re.IGNORECASE | re.MULTILINE)
+        last_ts = self.ts_list[-1]
+        found = False
+        for ts in ts_list:
+            if ts == last_ts:
+                found = True
+            elif found:
+                print "Append %s"%ts[1]
+                self.ts_list.append(ts)
+        #reactor.callLater(5,self.download_fragment)
+
+    def update_manifest_err(self,content):
+        return
+
+    def download_err(self,content):
+        self.deferred.errback("Error while downloading %s"%self.ts_list[self.ts_num][1])
+
+    def download_finished(self):
+        self.totalbytes = self.currentbytes
+        self.deferred.callback("Done")
+
+    def start(self):
+        reactor.callLater(1,self.start_download)
+        return self.deferred
+
+    def stop(self):
+        self.deferred.errback() # TODO
+
+    def addProgress(self, progress_callback):
+        print "[addProgress]"
+        self.progress_callback = progress_callback
+
+#####################################################################################################
+def get_header(url,headers=None):
+    headers = {"user-agent":USER_AGENT}
+    r = requests.head(url,headers=headers)
+    return r.headers
+
+def get_ext(mtype):
+    stype = "http"
+    if mtype in ("vnd.apple.mpegURL","application/x-mpegURL",'application/x-mpegurl',"application/vnd.apple.mpegurl"):
+        return ".ts","hls"
+    elif mtype in ("application/dash+xml"):
+        return ".ts","dash" # TODO dash stream type  could be different !
+    elif mtype in ("video/mp4"):
+        return ".mp4","http"
+    elif mtype in ("video/MP2T","video/mp2t"):
+        return ".ts","http"
+    elif mtype in ("video/x-flv"):
+        return ".flv","http"
+    elif mtype in ("video/quicktime"):
+        return ".mov","http"
+    elif mtype in ("video/x-msvideo"):
+        return ".avi","http"
+    elif mtype in ("video/x-ms-wmv"):
+        return ".wmv","http"
+    elif mtype in ("video/x-matroska"):
+        return ".mkv","http"
+    else:
+        return ".mp4","http"
+
+
+
+##############################################
+def print_progress(currentbytes, totalbytes):
+    progress = float(currentbytes)/float(totalbytes)*100
+    print "%s (%i/%i)"%(progress,currentbytes,totalbytes)
+
+def download_ok(*args):
+    print "Download OK"
+    reactor.stop()
+
+def download_err(e):
+    print "Download Error %s"%e.getBriefTraceback()
+    pass
+def stop():
+    reactor.stop()
+###############################################
+
+if __name__ == "__main__":
+    if len(sys.argv)>2:
+        url= sys.argv[1]
+        output = sys.argv[1]
+    else:
+        url = "http://walterebert.com/playground/video/hls/ts/480x270.m3u8"
+        url = "http://techslides.com/demos/sample-videos/small.mp4"
+        #url = "http://wx17.poiuytrew.pw/s/c507282042b1bf25e0b72c34a68426f3/hd_30/Jackie.2016.D.iTunes.BDRip.1080p_720.mp4"
+        url = "http://player.tvnet.lv/live/amlst:11/chunklist_w361981294_b528000.m3u8"
+        #url = "http://vod-hls-uk-live.akamaized.net/usp/auth/vod/piff_abr_full_hd/a3e90e-b08ktytr/vf_b08ktytr_f9d55583-afc7-49bb-9bf4-d8f1ac99f56f.ism.hlsv2.ism/vf_b08ktytr_f9d55583-afc7-49bb-9bf4-d8f1ac99f56f.ism.hlsv2-audio=128000-video=5070000.m3u8"
+        #url = "https://58174450afee9.streamlock.net/vod/mp4:_definst_/f/e/8e49fc32.mp4/playlist.m3u8?safwerwfasendtime=1490877870&safwerwfasstarttime=1490859339&safwerwfashash=hS2FfVZysQVazBQ6RJn1IhUevBkKxIF09Ly3BjfT43U="
+    try:
+        h = get_header(url,headers={"user-agent":"Enigma2"})
+        mtype = h.get("content-type")
+        ext,stream_type = get_ext(mtype)
+    except:
+        ext,stream_type = (".ts","hls")
+    output = urlparse.urlparse(url)[2].split('/')[-1] + ext
+    output = os.path.join("downloads", output)
+    if stream_type == "hls":
+        d = DownloadWithProgressFragmented(url,output,headers={"user-agent":"Enigma2"})
+    else:
+        d = DownloadWithProgress(url,output,headers={"user-agent":"Enigma2"})
+    d.addProgress(print_progress)
+    d.start().addCallback(download_ok).addErrback(download_err)
+    reactor.run()
+
+

resources/lib/sources/lmt.py

+#!/usr/bin/env python
+# coding=utf8
+#
+# This file is part of PlayStream - enigma2 plugin to play video streams from various sources
+# Copyright (c) 2016 ivars777 (ivars777@gmail.com)
+# Distributed under the GNU GPL v3. For full terms see http://www.gnu.org/licenses/gpl-3.0.en.html
+#
+try:
+    import json
+except:
+    import simplejson as json
+import requests
+import datetime, re, sys,os
+import ConfigParser
+from collections import OrderedDict
+from SourceBase import SourceBase
+import resolver
+try:
+    import util
+except:
+    sys.path.insert(0,'..')
+    import util
+
+headers2dict = lambda  h: dict([l.strip().split(": ") for l in h.strip().splitlines()])
+import HTMLParser
+h = HTMLParser.HTMLParser()
+
+class Source(SourceBase):
+
+    def __init__(self, country="",cfg_path=None):
+        self.name = "lmt"
+        self.title = "LMT straume"
+        self.img = "http://www.lob.lv/images/logo/lmt_straume_vert_rgb.png"
+        self.desc = "LMT straume - dažādi video latviesu valodā"
+        self.headers = headers2dict("""
+User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0
+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
+Accept-Language: en-US,en;q=0.5
+""")
+        self.headers2 = headers2dict("""
+User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0
+X-Requested-With: XMLHttpRequest
+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
+""")
+        self.url = "http://straume.lmt.lv/lv/"
+
+
+    ######### Entry point ########
+    def get_content(self, data):
+        print "[lmt] get_content:", data
+        source, data, path, plist, clist, params, qs = self.parse_data(data)
+        content = []
+        content.append(("..return", "back","","Return back"))
+
+        if clist=="home":
+            content.extend([
+                ("Meklēt", "lmt::meklet?q={0}","","Meklēt"),
+                ("Straumes", "lmt::straumes","","Tiešraides un aktuāli video"),
+                #("TV", "lmt::tv","","TV tiešraides (tikai LMT tīklā)"),
+                ("Jaunākie", "lmt::video/jaunakie?videoPage=1", "", "Visu žanru jaunākie video"),
+                ("Sports", "lmt::video/sports?videoPage=1", "", "Sports"),
+                ("Kultūra un māksla", "lmt::video/kultura?videoPage=1", "", "Kultūra un māksla"),
+                ("Konferences", "lmt::video/konferences?videoPage=1", "", "Konferences"),
+                ("Raidījumi", "lmt::video/raidijumi?videoPage=1", "", "Raidījumi"),
+                ("Notikumi", "lmt::video/notikumi?videoPage=1", "", "Notikumi"),
+                ("Filmas un seriāli", "lmt::video/filmas?videoPage=1", "", "Filmas un seriāli"),
+                ("Dažādi video", "lmt::video/dazadi?videoPage=1", "", "Dažādi video"),
+            ])
+            return content
+
+        elif clist in ("meklet","video", "straumes","video-saraksts"):
+            r=self.call(data)
+            result = re.findall('<a class="video-picture" (.+?)</li>', r, re.IGNORECASE | re.MULTILINE)
+            for r2 in result:
+                m = re.search('<a class="video-title" href="/lv/([^"]+)">([^<]+)<', r2)
+                title = m.group(2)
+                data2 = m.group(1)
+                m = re.search("([^ ]+) 2x", r2)
+                if m:
+                    img = m.group(1)
+                else:
+                    m = re.search('<img src="([^"]+)', r2)
+                    img = m.group(1) if m else ""
+                m = re.search('<span class="playlist-overlay">([^<]+)</span>', r2)
+                overlay = m.group(1) if m else ""
+                m = re.search('<span class="badge badge-[^>]+>([^<]+)(<[^>]+>([^<]+))*</span>', r2, re.IGNORECASE)
+                badge = ""
+                if m:
+                    badge = m.group(1)
+                    if m.group(3):
+                        badge = badge + m.group(3)
+                categories = re.findall('<span class="category-title">([^<]+)</span>', r2)
+                categories = "".join(categories)
+                if overlay:
+                    title = "%s [%s]"%(title,overlay)
+                if badge:
+                    title = "%s [%s]"%(title,badge)
+                desc = title
+                if categories:
+                    desc = desc + "\n"+ categories
+                content.append((title,self.name+"::"+data2,img,desc))
+            m = re.search("videoPage=(\d+)",data)
+            if m:
+                page = int(m.group(1))+1
+                data2 = re.sub(r"videoPage=\d+", r"videoPage=%s"%page, data)
+                content.append(("Next page",self.name+"::"+data2,self.img,"Next page"))
+            #print content
+            return content
+
+
+        ### kaut kas neparedzets ###
+        else:
+            return content
+
+    def is_video(self,data):
+        source,data,path,plist,clist,params,qs = self.parse_data(data)
+        if not clist in ("meklet","video", "straumes","video-saraksts","home"):
+            return True
+
+    def call(self, data,params=None,headers=None,lang=""):
+        if not headers: headers = self.headers
+        url = self.url+data
+        r = requests.get(url,headers = headers)
+        return r.content
+        #result = self._http_request(url,params,headers=headers)
+        return result
+
+    def get_streams(self,data):
+        print "[lmt] get_streams:", data
+        if not self.is_video(data):
+            return []
+        source,data,path,plist,clist,params,qs = self.parse_data(data)
+        r = self.call(path)
+        title = re.search("<h1>(.+?)</h1", r, re.IGNORECASE).group(1)
+        m = re.search('<a class="category-title".+?[^>]+>([^<]+)<', r, re.IGNORECASE | re.DOTALL)
+        categories = m.group(1) if m else ""
+        m = re.search('<span class="category-title">([^<]+)</span>.+?</p>', r, re.IGNORECASE | re.DOTALL)
+        if m:
+            categories = categories + m.group(1)
+        if categories:
+            tite = "%s [%s]"%(title,categories)
+        img = re.search('<meta property="twitter:image" content="([^"]+)">', r, re.IGNORECASE | re.DOTALL).group(1)
+        desc = title + "\n" + re.search('<meta property="og:description" content="([^"]+)">', r, re.IGNORECASE | re.DOTALL).group(1)
+        m = re.search('file: "([^"]+)"', r, re.IGNORECASE)
+        if m:
+            data2 = m.group(1)
+            stream = util.item()
+            stream["name"] = title
+            stream["url"] = data2
+            stream["img"] = img
+            stream["desc"] = desc
+            stream["resolver"] = "lmt"
+            return [stream]
+        else:
+            m = re.search('src="(http://www.youtube.com/[^"]+)"',r)
+            if m:
+                url = m.group(1)
+            else:
+                raise Exception("No stream found")
+            streams = resolver.resolve(url)
+            for s in streams:
+                s["name"] = title
+                s["desc"] = desc
+                s["img"] = img
+                streams.append(s)
+            return streams
+
+
+if __name__ == "__main__":
+    country= "lv"
+    c = Source(country)
+    if len(sys.argv)>1:
+        data= sys.argv[1]
+    else:
+        data = "home"
+    content = c.get_content(data)
+    for item in content:
+        print item
+    #cat = api.get_categories(country)
+    #chan = api.get_channels("lv")
+    #prog = api.get_programs(channel=6400)
+    #prog = api.get_programs(category=55)
+    #seas = api.get_seasons(program=6453)
+    #str = api.get_streams(660243)
+    #res = api.get_videos(802)
+    #formats = api.getAllFormats()
+    #det = api.detailed("1516")
+    #vid = api.getVideos("13170")
+    pass