Merge branch 'master' of http://git.blue.lv/home/content

ContentSources.py

                 cfg.add_item("home",(title,"%s::home"%pl,img,desc))
                 cfg.write_streams()
 
-    def get_content(self,data):
+    def get_content(self,data, qs=None):
         source = data.split("::")[0]
         if source in self.plugins:
             content0 = self.plugins[source].get_content(data)
         else:
             return self.error_content
 
-    def get_streams(self,data):
+    def get_streams(self,data, qs=None):
         if stream_type(data):
             if "::" in data:
                 data = data.split("::")[1]
         else:
             return []
 
-    def get_info(self,data):
+    def get_info(self,data, qs=None):
         nfo = {}
         if self.is_video(data):
             source = data.split("::")[0]
         return nfo
 
 
-    def stream_type(self,data):
+    def stream_type(self,data, qs=None):
         return stream_type(data)
 
-    def is_video(self,data):
+    def is_video(self,data, qs=None):
         if self.stream_type(data):
             return True
         source = data.split("::")[0]
         else:
             return False
 
-    def options_read(self,source):
+    def options_read(self,source, qs=None):
         if source in self.plugins:
             options = self.plugins[source].options_read()
             if options:
         else:
             return None
 
-    def options_write(self,source,options):
+    def options_write(self,source,options, qs=None):
         if source in self.plugins:
             return self.plugins[source].options_write(options)
         else:

changelog.md

+**23.02.2019**
+- salabots tvdom
+- playstreamproxy servē strīmus priekš m3u8
+
 **23.12.2018**
 - salabots tvplay, filmix
 

playstreamproxy.py

 # -*- coding: utf-8 -*-
 """
 StreamProxy daemon (based on Livestream daemon)
-Ensures persistent cookies, User-Agents and others tricks to play protected HLS/DASH streams
+Provides API to ContetSources + stream serving to play via m3u8 playlists
 """
 import os
 import sys
 import time
 import atexit
-import re
+import re, json
 import binascii
 
 from signal import SIGTERM
 import urllib,urlparse
 #import cookielib,urllib2
 import requests
+from ContentSources import ContentSources
+from sources.SourceBase import stream_type
+import util
+from util import streamproxy_decode3, streamproxy_encode3
 
 try:
     from requests.packages.urllib3.exceptions import InsecureRequestWarning
 
 HOST_NAME = ""
 PORT_NUMBER = 8880
-DEBUG = True
+DEBUG = False
 DEBUG2 = False
 
 SPLIT_CHAR = "~"
 SPLIT_CODE = "%7E"
 EQ_CODE = "%3D"
 COL_CODE = "%3A"
+cunicode = lambda s: s.decode("utf8") if isinstance(s, str) else s
+cstr = lambda s: s.encode("utf8") if isinstance(s, unicode) else s
 headers2dict = lambda  h: dict([l.strip().split(": ") for l in h.strip().splitlines()])
+
 headers0 = headers2dict("""
-icy-metadata: 1
 User-Agent: GStreamer souphttpsrc libsoup/2.52.2
+icy-metadata: 1
 """)
-sessions = {}
+headers0_ = headers2dict("""
+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
+User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36
+""")
+
 cur_directory = os.path.dirname(os.path.realpath(__file__))
-sources = None
 slinks = {}
+sessions = {}
 
+cfg_file = "streams.cfg"
+sources = ContentSources("sources", cfg_file)
 
 class StreamHandler(BaseHTTPRequestHandler):
 
 
     def do_GET(self):
         """Respond to a GET request"""
-        print "\n\n"+40*"#"+"\nget_url: \n%s", self.path
-        p = self.path.split("~")
-        #url = urllib.unquote(p[0][1:]) # TODO - vajag nocekot vai visi urli strādā
-        urlp = p[0][1:]
-        url = urlp.replace(COL_CODE, ":")
-        #headers = self.headers.dict
-        headers = {} # TODO izmanto saņemtos headerus, var aizvietot ar defaultajiem
-        #headers["host"] = urlparse.urlparse(url).hostname
-        if len(p)>1:
-            for h in p[1:]:
-                k = h.split("=")[0].lower()
-                v = urllib.unquote(h.split("=")[1])
-                headers[k]=v
+        #
+        print "\n\n"+40*"#"+"\nget_url: \n%s" % self.path
+        cmd = self.path.split("/")[1]
         if DEBUG:
-            print "url=%s"%url
+            print "cmd=%s"%cmd
             print "Original request headers + url headers:"
             print_headers(self.headers.dict)
         self.protocol_version = 'HTTP/1.1'
-
         try:
-            if "::" in url: # encoded source link
-                self.fetch_source(urlp, headers)
-            elif ".lattelecom.tv/" in url: # lattelecom.tv hack
-                self.fetch_ltc( url, headers)
-            elif "filmas.lv" in url or "viaplay" in url: #  HLS session/decode filmas.lv in url:
-                self.fetch_url2(url, headers)
-            else: # plain fetch
-                self.fetch_url( url, headers)
+            if cmd == "playstream":
+                self.fetch_source( self.path)
+            elif cmd in ["get_content", "get_streams", "get_info", "is_video", "options_read", "options_write"]:
+                cmd, data, headers, qs = streamproxy_decode3(self.path)
+                if cmd == "get_content":
+                    content = sources.get_content(data)
+                elif cmd == "get_streams":
+                    content = sources.get_streams(data)
+                elif cmd == "get_info":
+                    content = sources.get_info(data)
+                elif cmd == "is_video":
+                    content = sources.is_video(data)
+                elif cmd == "options_read":
+                    content = sources.options_read(data)
+                else:
+                    content = []
+                txt = json.dumps(content)
+                self.send_response(200)
+                self.send_header("Server", "playstreamproxy")
+                self.send_header("Content-type", "application/json")
+                self.end_headers()
+                self.wfile.write(txt)
+                self.wfile.close()
+            else:
+                self.write_error(404)
+
         except Exception as e:
             print "Got Exception: ", str(e)
             import traceback
             traceback.print_exc()
 
+
     ### Remote server request procedures ###
 
     def fetch_offline(self):
         self.wfile.write(open("offline.mp4", "rb").read())
         #self.wfile.close()
 
-    def fetch_source(self, urlp, headers):
+    def fetch_source(self, urlp):
+        cmd, data, headers, qs = streamproxy_decode3(urlp)
         if DEBUG:
             print "\n***********************************************************"
             print "fetch_source: \n%s"%urlp
         base_data = hls_base(urlp)
-        data = urllib.unquote_plus(base_data)[:-1]
-        if DEBUG: print "base_data=", base_data
-        if DEBUG: print "data=", data
-        if not base_data in slinks :
+        if DEBUG:
+            print "base_data=", base_data
+            print "data=", data
+        if not base_data in slinks:
             streams = sources.get_streams(data)
             if not streams:
                 self.write_error(500)  # TODO
                 return
-            url = streams[0]["url"]
-            base_url = hls_base(url)
+            stream = streams[0]
+            url = stream["url"]
+            headers = stream["headers"] if "headers" in stream else headers0
+            base_url = hls_base2(url)
             if DEBUG: print "New link, base_url=",base_url
             ses = requests.Session()
             ses.trust_env = False
             else:
                 url = urlp.replace(base_data, slinks[base_data]["base_url"])
                 if DEBUG: print "Existing new link", url
-        r = self.get_page_ses(url,ses,True,headers = headers)
+        headers2 = headers if headers else self.headers.dict
+        headers2 = del_headers(headers2, ["host"])
+        r = self.get_page_ses(url,ses,True,headers = headers2)
         code = r.status_code
-        if not code in (200,206): # TODO mēģina vēlreiz get_streams
+        if not code in (200,206): # TODO 206 apstrāde!
             self.write_error(code)
             return
+        if code == 206:
+            print "Code=206"
         self.send_response(code)
+        #headers2 = del_headers(r.headers, ["Content-Encoding"])
         self.send_headers(r.headers)
         CHUNK_SIZE = 1024 *4
-        for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
-            try:
-                self.wfile.write(chunk)
-            except Exception as e:
-                print "Exception: ", str(e)
-                self.wfile.close()
-                return
-        if DEBUG: print "**File downloaded"
-        if "connection" in r.headers and r.headers["connection"] <> "keep-alive":
-            self.wfile.close()
-        return
-
-
-    def fetch_url(self, url,headers):
-        if DEBUG:
-            print "\n***********************************************************"
-            print "fetch_url: \n%s"%url
-        r = self.get_page(url,headers = headers)
-        code = r.status_code
-        if not code in (200,206):
-            self.write_error(code)
-            return
-        self.send_response(code)
-        self.send_headers(r.headers)
-        CHUNK_SIZE = 1024*4
-        for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
+        while True:
+            chunk = r.raw.read(CHUNK_SIZE, decode_content=False)
+            if not chunk:
+                break
             try:
                 self.wfile.write(chunk)
             except Exception as e:
                 self.wfile.close()
                 return
         if DEBUG: print "**File downloaded"
-        if "connection" in r.headers and r.headers["connection"] <> "keep-alive":
-            self.wfile.close()
-        return
-
-    def fetch_ltc(self, url, headers):
-        "lattelecom.tv hack (have to update chunklist after each 6 min"
-        if DEBUG:
-            print "\n\n***********************************************************"
-            print "fetch_ltc: \n%s"%url
-        base_url = hls_base(url)
-        if DEBUG: print "base_url=",base_url
-        if base_url not in sessions:
-            if DEBUG: print "New session"
-            sessions[base_url] = {}
-            sessions[base_url]["session"] = requests.Session()
-            sessions[base_url]["session"].trust_env = False
-            sessions[base_url]["session"].headers.update(headers0)
-            sessions[base_url]["playlist"] = ""
-            sessions[base_url]["chunklist"] = []
-
-        #  change ts file to valid one media_w215689190_33.ts?
-        tsfile = re.search("media_\w+_(\d+)\.ts", url, re.IGNORECASE)
-        if tsfile and sessions[base_url]["chunklist"]:
-            tnum = int(tsfile.group(1))
-            url2 = sessions[base_url]["chunklist"][tnum]
-            if not url2.startswith("http"):
-                url2 = base_url + url2
-            url = url2
-            if DEBUG: print "[playstreamproxy] url changed to ", url
-
-        ### get_page ###
-        ses = sessions[base_url]["session"]
-        #ses.headers.update(headers0)
-        ses.headers.update(headers)
-        # ses.headers["Connection"]="Keep-Alive"
-        r = self.get_page_ses(url,ses)
-        code = r.status_code #r.status_code
-
-        if not (code in (200,206)) and tsfile:
-            # update chunklist
-            r2 = self.get_page(sessions[base_url]["playlist"])
-            streams = re.findall(r"#EXT-X-STREAM-INF:.*?BANDWIDTH=(\d+).*?\n(.+?)$", r2.content, re.IGNORECASE | re.MULTILINE)
-            if streams:
-                sorted(streams, key=lambda item: int(item[0]), reverse=True)
-                chunklist = streams[0][1]
-                if not chunklist.startswith("http"):
-                    chunklist = base_url + chunklist
-            else:
-                self.write_error(r.status_code)
-                return
-            print "[playstreamproxy] trying to update chunklist", chunklist
-            r3 = self.get_page_ses(chunklist,ses,True)
-            ts_list = re.findall(r"#EXTINF:.*?\n(.+?)$", r3.content, re.IGNORECASE | re.MULTILINE)
-            sessions[base_url]["chunklist"]= ts_list
-            tnum = int(tsfile.group(1))
-            url2 = sessions[base_url]["chunklist"][tnum]
-            if not url2.startswith("http"):
-                url2 = base_url + url2
-            r = self.get_page_ses(url2,ses,True)
-            if not r.status_code in (200,206):
-                self.write_error(r.status_code)
-                return
-        elif not r.status_code in (200,206):
-            self.write_error(r.status_code)
-            return
-
-        if "playlist.m3u8" in url:
-            sessions[base_url]["playlist"] = url
-
-        ### Start of return formin and sending
-        self.send_response(200)
-        #headers2 = del_headers(r.headers,["Content-Encoding",'Transfer-Encoding',"Connection",'content-range',"range"])
-        headers2  = {"server":"playstreamproxy", "content-type":"text/html"}
-
-        if DEBUG: print "\n** Return  content"
-        headers2["content-type"]  = r.headers["content-type"]
-        if "content-length" in r.headers:
-            headers2["content-length"] = r.headers["content-length"]
-        self.send_headers(r.headers)
-        CHUNK_SIZE = 4 * 1024
-        for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
-            try:
-                #print "#",
-                self.wfile.write(chunk)
-            except Exception as e:
-                print "Exception: ", str(e)
-                return
-        if DEBUG: print "File downloaded = "
+        #if "connection" in r.headers and r.headers["connection"] <> "keep-alive":
         self.wfile.close()
-        #time.sleep(1)
         return
 
 
-    def fetch_url2(self, url, headers):
-        if DEBUG:
-            print "\n***********************************************************"
-            print "fetch_url2: \n%s"%url
-        base_url = hls_base(url)
-        if DEBUG: print "base_url=",base_url
-        if base_url not in sessions:
-            if DEBUG: print "New session"
-            sessions[base_url] = {}
-            sessions[base_url]["session"] = requests.Session()
-            sessions[base_url]["session"].trust_env = False
-            sessions[base_url]["session"].headers.update(headers0)
-            sessions[base_url]["key"] = binascii.a2b_hex(headers["key"]) if "key" in headers and headers["key"] else None
-        ses = sessions[base_url]["session"]
-        ses.trust_env = False
-        key = sessions[base_url]["key"]
-        #ses.headers.clear()
-        ses.headers.update(headers)
-        r = self.get_page_ses(url, ses,stream=False)
-        code = r.status_code #r.status_code
-        if not (code in (200,206)):
-            self.write_error(r.status_code)
-            return
-
-        ### Start of return formin and sending
-        self.send_response(200)
-        #headers2 = del_headers(r.headers,["Content-Encoding",'Transfer-Encoding',"Connection",'content-range',"range"])
-        headers2  = {"server":"playstreamproxy", "content-type":"text/html"}
-
-        # Content-Type: application/vnd.apple.mpegurl (encrypted)
-        if r.headers["content-type"] == "application/vnd.apple.mpegurl" and key:
-            content = r.content
-            content = r.content.replace(base_url,"")
-            content = re.sub("#EXT-X-KEY:METHOD=AES-128.+\n", "", content, 0, re.IGNORECASE | re.MULTILINE)
-            headers2["content-type"] = "application/vnd.apple.mpegurl"
-            headers2["content-length"] = "%s"%len(content)
-            r.headers["content-length"] = "%s"%len(content)
-            #headers2['content-range'] = 'bytes 0-%s/%s'%(len(content)-1,len(content))
-            self.send_headers(headers2)
-            #self.send_headers(r.headers)
-            self.wfile.write(content)
-            self.wfile.close()
-
-        # Content-Type: video/MP2T (encrypted)
-        elif r.headers["content-type"] == "video/MP2T" and key:
-            print "Decode video/MP2T"
-            content = r.content
-            from Crypto.Cipher import AES
-            iv = content[:16]
-            d = AES.new(key, AES.MODE_CBC, iv)
-            content = d.decrypt(content[16:])
-            headers2["content-type"] = "video/MP2T"
-            headers2["content-length"] = "%s"% (len(content))
-            #headers2['content-range'] = 'bytes 0-%s/%s' % (len(content) - 1, len(content))
-            print content[0:16]
-            print "Finish decode"
-            self.send_headers(headers2)
-            self.wfile.write(content)
-            self.wfile.close()
-
-        else:
-            if DEBUG: print "Return regular content"
-            headers2["content-type"]  = r.headers["content-type"]
-            if "content-length" in r.headers:
-                headers2["content-length"] = r.headers["content-length"]
-            self.send_headers(r.headers)
-            #self.send_headers(headers2)
-            CHUNK_SIZE = 4 * 1024
-            for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
-                try:
-                    #print "#",
-                    self.wfile.write(chunk)
-                except Exception as e:
-                    print "Exception: ", str(e)
-                    return
-            if DEBUG: print "File downloaded = "
-            if "connection" in r.headers and r.headers["connection"]<>"keep-alive":
-                self.wfile.close()
-            #time.sleep(1)
-            return
-
     def send_headers(self,headers):
         #if DEBUG:
             #print "**Return headers: "
     """Handle requests in a separate thread."""
 
 def start(host = HOST_NAME, port = PORT_NUMBER):
-    import ContentSources, util
-    global sources
-    sources = ContentSources.ContentSources(os.path.join(cur_directory, "sources"))
     httpd = ThreadedHTTPServer((host, port), StreamHandler)
     print time.asctime(), "Server Starts - %s:%s" % (HOST_NAME, PORT_NUMBER)
     try:
     return headers
 
 def hls_base(url):
-    url2 = url.split("?")[0]
-    url2 = "/".join(url2.split("/")[0:-1])+ "/"
-    return url2
+    base = url.split("?")[0]
+    base = "/".join(base.split("/")[0:3])+ "/"
+    rest = url.replace(base, "")
+    return base
+
+def hls_base2(url):
+    base = url.split("?")[0]
+    base = "/".join(base.split("/")[0:-1])+ "/"
+    rest = url.replace(base, "")
+    return base
+
+print streamproxy_encode3("get_content", "ltc::home", qs={"user": "ivars777", "password": "Kaskade7"})
+print streamproxy_encode3("get_content", "ltc::content/live-streams/101?include=quality", qs={"user": "ivars777", "password": "Kaskade7"})
+urlp = streamproxy_encode3("playstream",
+                          "ltc::content/live-streams/101?include=quality",
+                          #headers=headers0,
+                          qs={"user": "ivars777", "password": "Kaskade7"})
+print urlp
+print streamproxy_encode3("playstream", "replay::tiesraide/ltv1")
+print streamproxy_encode3("playstream", "tvdom::tiesraides/ltv1/")
+print streamproxy_encode3("playstream", "tvplay::asset/10311641")
+print streamproxy_encode3("playstream", "xtv::rigatv24/video/Zw4pVPqr7OX-festivals_mainam_pasauli_sakam_ar_sevi")
+print streamproxy_encode3("playstream", "iplayer::episodes/b094f49s")
+#cmd, data, headers, qs = streamproxy_decode3(urlp)
 
 if __name__ == "__main__":
     daemon = ProxyDaemon("/var/run/playstreamproxy.pid")

playstreamproxy0.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""
+StreamProxy daemon (based on Livestream daemon)
+Ensures persistent cookies, User-Agents and others tricks to play protected HLS/DASH streams
+"""
+import os
+import sys
+import time
+import atexit
+import re
+import binascii
+
+from signal import SIGTERM
+
+from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
+from SocketServer import ThreadingMixIn
+from urllib import unquote, quote
+import urllib,urlparse
+#import cookielib,urllib2
+import requests
+
+try:
+    from requests.packages.urllib3.exceptions import InsecureRequestWarning
+    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+except:
+    pass
+
+HOST_NAME = ""
+PORT_NUMBER = 8880
+DEBUG = True
+DEBUG2 = False
+
+SPLIT_CHAR = "~"
+SPLIT_CODE = "%7E"
+EQ_CODE = "%3D"
+COL_CODE = "%3A"
+headers2dict = lambda  h: dict([l.strip().split(": ") for l in h.strip().splitlines()])
+headers0 = headers2dict("""
+icy-metadata: 1
+User-Agent: GStreamer souphttpsrc libsoup/2.52.2
+""")
+sessions = {}
+cur_directory = os.path.dirname(os.path.realpath(__file__))
+sources = None
+slinks = {}
+
+
+class StreamHandler(BaseHTTPRequestHandler):
+
+    def do_HEAD(self):
+        print "**get_head"
+        self.send_response(200)
+        self.send_header("Server", "playstreamproxy")
+        if ".m3u8" in self.path.lower():
+            ct = "application/vnd.apple.mpegurl"
+        elif ".ts" in self.path.lower():
+            ct = "video/MP2T"
+        elif ".mp4" in self.path.lower():
+            ct = "video/mp4"
+        else:
+            ct = "text/html"
+        self.send_header("Content-type", ct)
+        self.end_headers()
+
+    def do_GET(self):
+        """Respond to a GET request"""
+        print "\n\n"+40*"#"+"\nget_url: \n%s", self.path
+        p = self.path.split("~")
+        #url = urllib.unquote(p[0][1:]) # TODO - vajag nocekot vai visi urli strādā
+        urlp = p[0][1:]
+        url = urlp.replace(COL_CODE, ":")
+        #headers = self.headers.dict
+        headers = {} # TODO izmanto saņemtos headerus, var aizvietot ar defaultajiem
+        #headers["host"] = urlparse.urlparse(url).hostname
+        if len(p)>1:
+            for h in p[1:]:
+                k = h.split("=")[0].lower()
+                v = urllib.unquote(h.split("=")[1])
+                headers[k]=v
+        if DEBUG:
+            print "url=%s"%url
+            print "Original request headers + url headers:"
+            print_headers(self.headers.dict)
+        self.protocol_version = 'HTTP/1.1'
+
+        try:
+            if "::" in url: # encoded source link
+                self.fetch_source(urlp, headers)
+            elif ".lattelecom.tv/" in url: # lattelecom.tv hack
+                self.fetch_ltc( url, headers)
+            elif "filmas.lv" in url or "viaplay" in url: #  HLS session/decode filmas.lv in url:
+                self.fetch_url2(url, headers)
+            else: # plain fetch
+                self.fetch_url( url, headers)
+        except Exception as e:
+            print "Got Exception: ", str(e)
+            import traceback
+            traceback.print_exc()
+
+    ### Remote server request procedures ###
+
+    def fetch_offline(self):
+        print "** Fetch offline"
+        self.send_response(200)
+        self.send_header("Server", "playstreamproxy")
+        self.send_header("Content-type", "video/mp4")
+        self.end_headers()
+        self.wfile.write(open("offline.mp4", "rb").read())
+        #self.wfile.close()
+
+    def fetch_source(self, urlp, headers):
+        if DEBUG:
+            print "\n***********************************************************"
+            print "fetch_source: \n%s"%urlp
+        base_data = hls_base(urlp)
+        data = urllib.unquote_plus(base_data)[:-1]
+        if DEBUG: print "base_data=", base_data
+        if DEBUG: print "data=", data
+        if not base_data in slinks :
+            streams = sources.get_streams(data)
+            if not streams:
+                self.write_error(500)  # TODO
+                return
+            url = streams[0]["url"]
+            base_url = hls_base(url)
+            if DEBUG: print "New link, base_url=",base_url
+            ses = requests.Session()
+            ses.trust_env = False
+            slinks[base_data] = {"data": data, "urlp":urlp,"url": url, "base_url": base_url,"session":ses}
+        else:
+            ses = slinks[base_data]["session"]
+            if urlp == slinks[base_data]["urlp"]:
+                url = slinks[base_data]["url"]
+                if DEBUG: print "Existing base link", url
+            else:
+                url = urlp.replace(base_data, slinks[base_data]["base_url"])
+                if DEBUG: print "Existing new link", url
+        r = self.get_page_ses(url,ses,True,headers = headers)
+        code = r.status_code
+        if not code in (200,206): # TODO mēģina vēlreiz get_streams
+            self.write_error(code)
+            return
+        self.send_response(code)
+        self.send_headers(r.headers)
+        CHUNK_SIZE = 1024 *4
+        for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
+            try:
+                self.wfile.write(chunk)
+            except Exception as e:
+                print "Exception: ", str(e)
+                self.wfile.close()
+                return
+        if DEBUG: print "**File downloaded"
+        if "connection" in r.headers and r.headers["connection"] <> "keep-alive":
+            self.wfile.close()
+        return
+
+
+    def fetch_url(self, url,headers):
+        if DEBUG:
+            print "\n***********************************************************"
+            print "fetch_url: \n%s"%url
+        r = self.get_page(url,headers = headers)
+        code = r.status_code
+        if not code in (200,206):
+            self.write_error(code)
+            return
+        self.send_response(code)
+        self.send_headers(r.headers)
+        CHUNK_SIZE = 1024*4
+        for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
+            try:
+                self.wfile.write(chunk)
+            except Exception as e:
+                print "Exception: ", str(e)
+                self.wfile.close()
+                return
+        if DEBUG: print "**File downloaded"
+        if "connection" in r.headers and r.headers["connection"] <> "keep-alive":
+            self.wfile.close()
+        return
+
+    def fetch_ltc(self, url, headers):
+        "lattelecom.tv hack (have to update chunklist after each 6 min"
+        if DEBUG:
+            print "\n\n***********************************************************"
+            print "fetch_ltc: \n%s"%url
+        base_url = hls_base(url)
+        if DEBUG: print "base_url=",base_url
+        if base_url not in sessions:
+            if DEBUG: print "New session"
+            sessions[base_url] = {}
+            sessions[base_url]["session"] = requests.Session()
+            sessions[base_url]["session"].trust_env = False
+            sessions[base_url]["session"].headers.update(headers0)
+            sessions[base_url]["playlist"] = ""
+            sessions[base_url]["chunklist"] = []
+
+        #  change ts file to valid one media_w215689190_33.ts?
+        tsfile = re.search("media_\w+_(\d+)\.ts", url, re.IGNORECASE)
+        if tsfile and sessions[base_url]["chunklist"]:
+            tnum = int(tsfile.group(1))
+            url2 = sessions[base_url]["chunklist"][tnum]
+            if not url2.startswith("http"):
+                url2 = base_url + url2
+            url = url2
+            if DEBUG: print "[playstreamproxy] url changed to ", url
+
+        ### get_page ###
+        ses = sessions[base_url]["session"]
+        #ses.headers.update(headers0)
+        ses.headers.update(headers)
+        # ses.headers["Connection"]="Keep-Alive"
+        r = self.get_page_ses(url,ses)
+        code = r.status_code #r.status_code
+
+        if not (code in (200,206)) and tsfile:
+            # update chunklist
+            r2 = self.get_page(sessions[base_url]["playlist"])
+            streams = re.findall(r"#EXT-X-STREAM-INF:.*?BANDWIDTH=(\d+).*?\n(.+?)$", r2.content, re.IGNORECASE | re.MULTILINE)
+            if streams:
+                sorted(streams, key=lambda item: int(item[0]), reverse=True)
+                chunklist = streams[0][1]
+                if not chunklist.startswith("http"):
+                    chunklist = base_url + chunklist
+            else:
+                self.write_error(r.status_code)
+                return
+            print "[playstreamproxy] trying to update chunklist", chunklist
+            r3 = self.get_page_ses(chunklist,ses,True)
+            ts_list = re.findall(r"#EXTINF:.*?\n(.+?)$", r3.content, re.IGNORECASE | re.MULTILINE)
+            sessions[base_url]["chunklist"]= ts_list
+            tnum = int(tsfile.group(1))
+            url2 = sessions[base_url]["chunklist"][tnum]
+            if not url2.startswith("http"):
+                url2 = base_url + url2
+            r = self.get_page_ses(url2,ses,True)
+            if not r.status_code in (200,206):
+                self.write_error(r.status_code)
+                return
+        elif not r.status_code in (200,206):
+            self.write_error(r.status_code)
+            return
+
+        if "playlist.m3u8" in url:
+            sessions[base_url]["playlist"] = url
+
+        ### Start of return formin and sending
+        self.send_response(200)
+        #headers2 = del_headers(r.headers,["Content-Encoding",'Transfer-Encoding',"Connection",'content-range',"range"])
+        headers2  = {"server":"playstreamproxy", "content-type":"text/html"}
+
+        if DEBUG: print "\n** Return  content"
+        headers2["content-type"]  = r.headers["content-type"]
+        if "content-length" in r.headers:
+            headers2["content-length"] = r.headers["content-length"]
+        self.send_headers(r.headers)
+        CHUNK_SIZE = 4 * 1024
+        for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
+            try:
+                #print "#",
+                self.wfile.write(chunk)
+            except Exception as e:
+                print "Exception: ", str(e)
+                return
+        if DEBUG: print "File downloaded = "
+        self.wfile.close()
+        #time.sleep(1)
+        return
+
+
+    def fetch_url2(self, url, headers):
+        if DEBUG:
+            print "\n***********************************************************"
+            print "fetch_url2: \n%s"%url
+        base_url = hls_base(url)
+        if DEBUG: print "base_url=",base_url
+        if base_url not in sessions:
+            if DEBUG: print "New session"
+            sessions[base_url] = {}
+            sessions[base_url]["session"] = requests.Session()
+            sessions[base_url]["session"].trust_env = False
+            sessions[base_url]["session"].headers.update(headers0)
+            sessions[base_url]["key"] = binascii.a2b_hex(headers["key"]) if "key" in headers and headers["key"] else None
+        ses = sessions[base_url]["session"]
+        ses.trust_env = False
+        key = sessions[base_url]["key"]
+        #ses.headers.clear()
+        ses.headers.update(headers)
+        r = self.get_page_ses(url, ses,stream=False)
+        code = r.status_code #r.status_code
+        if not (code in (200,206)):
+            self.write_error(r.status_code)
+            return
+
+        ### Start of return formin and sending
+        self.send_response(200)
+        #headers2 = del_headers(r.headers,["Content-Encoding",'Transfer-Encoding',"Connection",'content-range',"range"])
+        headers2  = {"server":"playstreamproxy", "content-type":"text/html"}
+
+        # Content-Type: application/vnd.apple.mpegurl (encrypted)
+        if r.headers["content-type"] == "application/vnd.apple.mpegurl" and key:
+            content = r.content
+            content = r.content.replace(base_url,"")
+            content = re.sub("#EXT-X-KEY:METHOD=AES-128.+\n", "", content, 0, re.IGNORECASE | re.MULTILINE)
+            headers2["content-type"] = "application/vnd.apple.mpegurl"
+            headers2["content-length"] = "%s"%len(content)
+            r.headers["content-length"] = "%s"%len(content)
+            #headers2['content-range'] = 'bytes 0-%s/%s'%(len(content)-1,len(content))
+            self.send_headers(headers2)
+            #self.send_headers(r.headers)
+            self.wfile.write(content)
+            self.wfile.close()
+
+        # Content-Type: video/MP2T (encrypted)
+        elif r.headers["content-type"] == "video/MP2T" and key:
+            print "Decode video/MP2T"
+            content = r.content
+            from Crypto.Cipher import AES
+            iv = content[:16]
+            d = AES.new(key, AES.MODE_CBC, iv)
+            content = d.decrypt(content[16:])
+            headers2["content-type"] = "video/MP2T"
+            headers2["content-length"] = "%s"% (len(content))
+            #headers2['content-range'] = 'bytes 0-%s/%s' % (len(content) - 1, len(content))
+            print content[0:16]
+            print "Finish decode"
+            self.send_headers(headers2)
+            self.wfile.write(content)
+            self.wfile.close()
+
+        else:
+            if DEBUG: print "Return regular content"
+            headers2["content-type"]  = r.headers["content-type"]
+            if "content-length" in r.headers:
+                headers2["content-length"] = r.headers["content-length"]
+            self.send_headers(r.headers)
+            #self.send_headers(headers2)
+            CHUNK_SIZE = 4 * 1024
+            for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
+                try:
+                    #print "#",
+                    self.wfile.write(chunk)
+                except Exception as e:
+                    print "Exception: ", str(e)
+                    return
+            if DEBUG: print "File downloaded = "
+            if "connection" in r.headers and r.headers["connection"]<>"keep-alive":
+                self.wfile.close()
+            #time.sleep(1)
+            return
+
+    def send_headers(self,headers):
+        #if DEBUG:
+            #print "**Return headers: "
+            #print_headers(headers)
+        for h in headers:
+            self.send_header(h, headers[h])
+        self.end_headers()
+
+    def write_error(self,code):
+        print "***Error, code=%s" % code
+        self.send_response(code)
+        #self.send_headers(r.headers)
+        self.wfile.close() # TODO?
+        # self.fetch_offline()
+
+    def get_page_ses(self,url,ses,stream=True, headers=None):
+        headers= headers if headers else headers0
+        ses.headers.update(headers)
+        if DEBUG:
+            print "\n\n====================================================\n**get_page_ses\n%s"%url
+            print "**Server request headers: "
+            print_headers(ses.headers)
+        r = ses.get(url, stream=stream, verify=False)
+        if DEBUG:
+            print "**Server response:", r.status_code
+            print "**Server response headers: "
+            print_headers(r.headers)
+        return r
+
+    def get_page(self,url,headers=None):
+        if not headers:
+            headers = headers0
+        if DEBUG:
+            print "\n\n====================================================\n**get_page\n%s"%url
+            print "**Server request headers: "
+            print_headers(headers)
+        r = requests.get(url, headers=headers,stream=True)
+        if DEBUG:
+            print "**Server response:", r.status_code
+            print "**Server response headers: "
+            print_headers(r.headers)
+        return r
+
+    def address_string(self):
+        host, port = self.client_address[:2]
+        #return socket.getfqdn(host)
+        return host
+
+class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
+    """Handle requests in a separate thread."""
+
+def start(host = HOST_NAME, port = PORT_NUMBER):
+    import ContentSources, util
+    global sources
+    sources = ContentSources.ContentSources(os.path.join(cur_directory, "sources"))
+    httpd = ThreadedHTTPServer((host, port), StreamHandler)
+    print time.asctime(), "Server Starts - %s:%s" % (HOST_NAME, PORT_NUMBER)
+    try:
+        httpd.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    httpd.server_close()
+    print time.asctime(), "Server Stops - %s:%s" % (HOST_NAME, PORT_NUMBER)
+
+
+class Daemon:
+    """
+    A generic daemon class.
+    Usage: subclass the Daemon class and override the run() method
+    """
+    def __init__(self, pidfile, stdin="/dev/null", stdout="/dev/null", stderr="/dev/null"):
+        self.stdin = stdin
+        self.stdout = stdout
+        self.stderr = stderr
+        self.pidfile = pidfile
+
+    def daemonize(self):
+        """
+        do the UNIX double-fork magic, see Stevens' "Advanced
+        Programming in the UNIX Environment" for details (ISBN 0201563177)
+        http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
+        """
+        try:
+            pid = os.fork()
+            if pid > 0:
+                # exit first parent
+                sys.exit(0)
+        except OSError, e:
+            sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
+            sys.exit(1)
+
+        # decouple from parent environment
+        os.chdir("/")
+        os.setsid()
+        os.umask(0)
+
+        # do second fork
+        try:
+            pid = os.fork()
+            if pid > 0:
+                # exit from second parent
+                sys.exit(0)
+        except OSError, e:
+            sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror))
+            sys.exit(1)
+
+        # redirect standard file descriptors
+        sys.stdout.flush()
+        sys.stderr.flush()
+        si = file(self.stdin, "r")
+        so = file(self.stdout, "a+")
+        se = file(self.stderr, "a+", 0)
+        os.dup2(si.fileno(), sys.stdin.fileno())
+        os.dup2(so.fileno(), sys.stdout.fileno())
+        os.dup2(se.fileno(), sys.stderr.fileno())
+
+        # write pidfile
+        atexit.register(self.delpid)
+        pid = str(os.getpid())
+        file(self.pidfile,"w+").write("%s\n" % pid)
+
+    def delpid(self):
+        os.remove(self.pidfile)
+
+    def start(self):
+        """
+        Start the daemon
+        """
+        # Check for a pidfile to see if the daemon already runs
+        try:
+            pf = file(self.pidfile,"r")
+            pid = int(pf.read().strip())
+            pf.close()
+        except IOError:
+            pid = None
+
+        if pid:
+            message = "pidfile %s already exist. Daemon already running?\n"
+            sys.stderr.write(message % self.pidfile)
+            sys.exit(1)
+
+        # Start the daemon
+        self.daemonize()
+        self.run()
+
+    def stop(self):
+        """
+        Stop the daemon
+        """
+        # Get the pid from the pidfile
+        try:
+            pf = file(self.pidfile,"r")
+            pid = int(pf.read().strip())
+            pf.close()
+        except IOError:
+            pid = None
+
+        if not pid:
+            message = "pidfile %s does not exist. Daemon not running?\n"
+            sys.stderr.write(message % self.pidfile)
+            return # not an error in a restart
+
+        # Try killing the daemon process
+        try:
+            while 1:
+                os.kill(pid, SIGTERM)
+                time.sleep(0.1)
+        except OSError, err:
+            err = str(err)
+            if err.find("No such process") > 0:
+                if os.path.exists(self.pidfile):
+                    os.remove(self.pidfile)
+            else:
+                print str(err)
+                sys.exit(1)
+
+    def restart(self):
+        """
+        Restart the daemon
+        """
+        self.stop()
+        self.start()
+
+    def run(self):
+        """
+        You should override this method when you subclass Daemon. It will be called after the process has been
+        daemonized by start() or restart().
+        """
+
+class ProxyDaemon(Daemon):
+    def run(self):
+        start()
+
+def print_headers(headers):
+    for h in headers:
+        print "%s: %s"%(h,headers[h])
+
+def del_headers(headers0,tags):
+    headers = headers0.copy()
+    for t in tags:
+        if t in headers:
+            del headers[t]
+        if t.lower() in headers:
+            del headers[t.lower()]
+    return headers
+
+def hls_base(url):
+    url2 = url.split("?")[0]
+    url2 = "/".join(url2.split("/")[0:-1])+ "/"
+    return url2
+
+if __name__ == "__main__":
+    daemon = ProxyDaemon("/var/run/playstreamproxy.pid")
+    if len(sys.argv) == 2:
+        if "start" == sys.argv[1]:
+            daemon.start()
+        elif "stop" == sys.argv[1]:
+            daemon.stop()
+        elif "restart" == sys.argv[1]:
+            daemon.restart()
+        elif "manualstart" == sys.argv[1]:
+            start()
+        else:
+            print "Unknown command"
+            sys.exit(2)
+        sys.exit(0)
+    else:
+        print "usage: %s start|stop|restart|manualstart" % sys.argv[0]
+        sys.exit(2)
+

playstreamproxy5.py

+# coding: utf-8
+u"""
+GIS failu web serveris
+"""
+prog = "web-server"
+version="%(prog)s v0.1"
+epilog = "(c)Ivars 2017"
+
+import sys, os, os.path, argparse
+import re,cgi, urllib, json
+import datetime,time
+
+from zope.interface import implements
+from twisted.internet import reactor,defer, threads, task
+from twisted.web import server, resource, http, error, util
+from twisted.python import log,logfile
+from twisted.web import static
+
+xset = lambda x,y: x if x else y
+
+options = []
+
+class ResponseFile(static.File):
+
+    def __init__(self, path):
+        static.File.__init__(self, path)
+
+    def render_GET(self, request):
+        #request.setHeader('Content-Disposition', ['attachment ; filename="tick_db_export.csv"'])
+        return static.File.render_GET(self, request)
+
+def parse_arguments(argv):
+    "Parse command line arguments"
+    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description =__doc__,epilog=epilog, version=version, prog=prog)
+    parser.add_argument("-p", "--port",  type=int, help="web server port")
+    parser.add_argument("--debug", action="store_true", dest="debug", default=False, help="print debug info")
+    return parser.parse_args(argv), parser
+
+#import config   # get parameters from ini file
+def main(argv):
+
+    # get global parameters from command line arguments and ini file
+    global options
+    options,parser = parse_arguments(argv[1:])
+
+    #log.startLogging(logfile.DailyLogFile.fromFullPath("logs\\gis-faili.log"))
+    log.startLogging(sys.stdout)
+
+    # root = MyFile(base)               # no directory browsing
+    #root = Root()
+    #root = static.File("/")
+    #root = ResponseFile(".")
+    root = urlproxy.URLProxy()
+
+    # Start web server
+    reactor.listenTCP(8000, server.Site(root))
+    reactor.run()
+
+
+class ResourceThread(resource.Resource):
+    "Resource subclass, which renders request in thread. Redefine 'process' "
+    def render_GET(self, request):
+        d = threads.deferToThread(self.process,request)
+        d.addCallback(self._cb,request)
+        d.addErrback(self._eb,request)
+        request.notifyFinish().addErrback(self._connection_lost, d)
+        return server.NOT_DONE_YET
+
+    def render_POST(self, request):
+        d = threads.deferToThread(self.process,request)
+        d.addCallback(self._cb,request)
+        d.addErrback(self._eb,request)
+        request.notifyFinish().addErrback(self._connection_lost, d)
+        return server.NOT_DONE_YET
+
+    def _cb(self, result, request):
+        "Callback function"
+        request.write(result)
+        request.finish()
+
+    def _eb(self, failure, request):
+        "Errorback function"
+        #err = failure.trap(defer.CancelledError)
+        if failure == defer.CancelledError:
+            log.msg("Connection canceled")
+            return None
+        else:
+            #log.err(failure)
+            failure.printTraceback()
+            body = resource.ErrorPage(http.INTERNAL_SERVER_ERROR,
+                             "Internal errror, see log file for details","").render(request)
+        request.write(body)
+        request.finish()
+
+    def _connection_lost(self,err,d):
+        d.cancel()
+
+    def process(self,request):
+        "Process request. Shoud be redefined"
+        return "No process method defined"
+
+    def get_req(self,request,exclude=[]):
+        "Return request parameters as Rec() object in utf8, add user. Exclude parameters listed in exclude"
+        req = Rec()
+        for k,v in request.args.iteritems():
+            if k in exclude:
+                continue
+            req[k] = ",".join(v)
+            req[k] = req[k].decode("utf8")
+        req.user = request.getUser()
+        return req
+
+
+######### http commands handlers ####################
+class Root(ResourceThread):
+    "Server root"
+    isLeaf = True
+
+    def process(self,request):
+        "Apstrada pieprasijumu"
+        req = self.get_req(request)
+        request.responseHeaders.setRawHeaders("Content-type",['text/html; charset=utf-8'])
+        html = u"Response text".encode("utf8")
+        return html
+        #self.file = static.File("./aaa.mp4")
+        #request.uri = "/aaa.mp4"
+        #return self.file.render_GET(request)
+
+
+import UserDict
+class Rec(UserDict.DictMixin):
+    "Dict like class allowing a.x instead of a['x']"
+    def __init__(self, dict=None, **kwargs):
+        self.__dict__ = {}
+        if dict is not None:
+            self.update(dict)
+        if len(kwargs):
+            self.update(kwargs)
+    def __setitem__(self, name, value): setattr(self, name, value)
+    def __getitem__(self, name): return getattr(self,name,None)
+    def __getattr__(self,key): return None
+    def has_key(self,key): return True if self.__dict__.has_key(key) else False
+    def keys(self): return list(self.__dict__)
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

run.py

                     print "No image found ", img_path
             elif img:
                 fcache = img.replace(":", "_").replace("/", "-").replace(":", "_")
+                fcache = fcache.split("?")[0]
                 fcache = os.path.join(self.tmp_path, fcache)
                 if os.path.exists(fcache):
                     im =  Image.open(fcache)

sources/tvdom.py

         elif clist=="play_arhivs" and len(data.split("/"))==3 and not re.search("_\d+",plist[2]):
             url = "https://tvdom.tv/"+data
             r = self._http_request(url)
-            vid=re.search('materialIdentifier : "(\d+)', r, re.DOTALL).group(1)
+            vid=re.search('materialIdentifier":"(\d+)', r, re.DOTALL).group(1)
             pname = re.split("[_\.\-]", plist[-1])[0]
             data2 = data+"_"+vid
             m = re.search(r'var program_title\s*= "(.+?)"', r)
             r2 = r if i2 == -1 else r[:i2]
             result = re.findall(r'<img src="([^"]+)" class="content-item__image" alt=""><a href="/([^"]+)" class="content-item__hide-info">.+?<h3 class="content-item__name">\s*<a href="#">\s*([^<]*)\s*<br>\s*([^<]*)\s*</a>\s*</h3>', r2, re.DOTALL)
             for item in result:
-                if pname not in item[1]:
-                    continue
+                #if pname not in item[1]:
+                #    continue
                 title = item[2] + " " + item[3].strip()
                 title =  h.unescape(title.decode("utf8")).encode("utf8")
                 img = "https://tvdom.tv"+item[0]
 
         # Tiešraides video
         else:
-            m = re.search("var streamConnectionUrl = '([^']+)'", r, re.DOTALL)
+            #m = re.search("var streamConnectionUrl = '([^']+)'", r, re.DOTALL)
+            m = re.search(r"sources: (\[.+?\]),", r, re.DOTALL)
             if m:
-                data2 = m.group(1)
+                js = json.loads(m.group(1))
+                data2 = js[0]["src"]
             else:
                 if "no_player_body" in r:
                     raise Exception("Lai skatītos maksas kanālus, nepieciešams iegādāties abonementu")
-                raise Exception("No stream found")
+                else:
+                    raise Exception("No stream found")
             m = re.search('title: "([^"]+)"', r, re.DOTALL)
             title = m.group(1) if m else data2
-            m = re.search('<div id="panel">([^<]+)<', r, re.DOTALL)
-            desc = m.group(1) if m else title
-            m = re.search('<div id="panel">([^<]+)<', r, re.DOTALL)
-            desc = m.group(1) if m else title
-            m = re.search('var promo_image *= "([^"]+)', r, re.DOTALL)
-            img = m.group(1) if m else ""
+            m = re.search('<div class="fp-descr">([^<]+)<', r, re.DOTALL)
+            desc = m.group(1).strip() if m else title
+            m = re.search('class="content-item content-item--on-air catchUpEvent".+?<img src="(.+?)"', r, re.DOTALL)
+            img = "https://tvdom.tv" + m.group(1) if m else ""
             stream = util.item()
             stream["name"] = title
             stream["url"] = data2

util.py

             headers[h.split("=")[0]]=urllib.unquote(h.split("=")[1])
     return url,headers
 
+
+def streamproxy_encode3(cmd, data, proxy_url=None, headers=None, qs=None):
+    PROXY_URL = "http://localhost:8880/"
+    #url2 = url.replace(SPLIT_CHAR,SPLIT_CODE).replace(":",COL_CODE).replace(" ",SPACE_CODE)
+    data2 = urllib.quote_plus(data)
+    if not proxy_url:
+        proxy_url = PROXY_URL
+    urlp = proxy_url + cmd + "/" + data2 + "/"
+    qs2 = []
+    if headers:
+        headers2 = []
+        for h in headers:
+            headers2.append("%s=%s"%(h,urllib.quote_plus(headers[h])))
+        headers2 = SPLIT_CHAR.join(headers2)
+        qs2.append("headers="+urllib.quote_plus(headers2))
+    if qs:
+        for qv in qs:
+            qs2.append(qv+"="+urllib.quote_plus(qs[qv]))
+    if qs2:
+        qs2 = "&".join(qs2)
+    if qs2:
+        urlp = urlp + "?" + qs2
+    return urlp
+
+def streamproxy_decode3(urlp):
+    m = re.search("http://[^/]+",urlp)
+    path = urlp.replace(m.group(0),"") if m else urlp
+    cmd = path.split("/")[1]
+    data = path.split("/")[2].split("?")[0]
+    data = urllib.unquote_plus(data)
+    qs = path.split("?")[1] if "?" in path else ""
+    qs2 = {}
+    headers = {}
+    if qs:
+        qs = qs.split("&")
+        for q in qs:
+            qk = q.split("=")[0]
+            if qk == "headers":
+                hh = urllib.unquote_plus(q.split("=")[1])
+                hh = hh.split(SPLIT_CHAR)
+                for h in hh:
+                    headers[h.split("=")[0]] = urllib.unquote_plus(h.split("=")[1])
+            else:
+                qv = urllib.unquote_plus(q.split("=")[1]) if "=" in q else ""
+                qs2[qk] = qv
+    return cmd, data, headers, qs2
+
+
 class Captions(object):
     def __init__(self,uri):
         self.uri = uri