darbs

changelog.md

@@ -1,5 +1,5 @@
 **08.07.2019**
-- izmaiņas gitā (autonoms projekts)
+- izmaiņas GITā (autonoms projekts)
 
 **23.02.2019**
 - salabots tvdom

ltcproxy.py

@@ -0,0 +1,573 @@
+#!/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Shortcut.lv proxy server
+
+usage: %s start|stop|restart|manualstart [options]
+    -p PORT         - port number
+    -s WSGI_SERVER  - wsgi server - wsgiref,cheroot,mtwsgi,waitress...
+    -d              - debug printout
+    -r              - remote debug mode (ptvsd)"""
+
+import os, sys, time
+import urllib,urlparse, urllib2, requests
+from urllib import unquote, quote
+import re, json
+import ConfigParser, getopt
+import arrow
+from diskcache import Cache
+import daemonize
+import bottle
+from bottle import Bottle, hook, response, route, request, run
+
+cunicode = lambda s: s.decode("utf8") if isinstance(s, str) else s
+cstr = lambda s: s.encode("utf8") if isinstance(s, unicode) else s
+headers2dict = lambda  h: dict([l.strip().split(": ") for l in h.strip().splitlines()])
+
+headers0 = headers2dict("""
+User-Agent: Shortcut.lv v2.9.1 / Dalvik/1.6.0 (Linux; U; Android 4.4.2; SM-G900FD Build/KOT49H)
+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
+""")
+url0 = "https://manstv.lattelecom.tv/api/v1.7/get/content/"
+
+cur_directory = os.path.dirname(os.path.realpath(__file__))
+cache_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "cache")
+if not os.path.exists(cache_dir):
+    os.mkdir(cache_dir)
+
+config = ConfigParser.ConfigParser()
+proxy_cfg_file = os.path.join(cur_directory, "ltcproxy.cfg")
+
+DEBUG = False
+PORT_NUMBER = 8881
+REDIRECT = False
+CACHE = True
+KEY = ["1234","3786"]
+SERVER = "wsgiref"
+WORKERS = 10
+LTC_USER = "ivars777"
+LTC_PASSWORD = "kaskade7"
+
+if not os.path.exists(proxy_cfg_file):
+    config.add_section("ltcproxy")
+    config.set("ltcproxy", "debug", DEBUG)
+    config.set("ltcproxy", "port", PORT_NUMBER)
+    config.set("ltcproxy", "redirect", REDIRECT)
+    config.set("ltcproxy", "cache", CACHE)
+    config.set("ltcproxy", "key", " ".join(KEY))
+    config.set("ltcproxy", "wsgi", SERVER)
+    config.set("ltcproxy", "workers", WORKERS)
+    config.set("ltcproxy", "ltc_user", LTC_USER)
+    config.set("ltcproxy", "ltc_password", LTC_PASSWORD)
+    config.write(open(proxy_cfg_file, "w"))
+else:
+    config.read(proxy_cfg_file)
+    DEBUG = config.getboolean("ltcproxy", "debug")
+    PORT_NUMBER = config.getint("ltcproxy", "port")
+    REDIRECT = config.getboolean("ltcproxy", "redirect")
+    CACHE = config.getboolean("ltcproxy", "cache")
+    KEY = config.get("ltcproxy", "key").split(" ")
+    SERVER = config.get("ltcproxy", "wsgi")
+    WORKERS = config.getint("ltcproxy", "workers")
+    LTC_USER = config.get("ltcproxy", "ltc_user")
+    LTC_PASSWORD = config.get("ltcproxy", "ltc_password")
+
+s = Cache(cache_dir)
+app = Bottle()
+token = None
+
+########################################################################################
+
+@app.hook('before_request')
+def set_globals():
+    global s, headers0, token
+    key = request.path.split("/")[1]
+    if not key in KEY:
+        print "Error: Wrong key - %s"% key
+        raise bottle.HTTPError(500, "Wrong key")
+    s = Cache(cache_dir)
+    if "token" in s and s["token"]:
+        token = s["token"]
+    else:
+        token = login(LTC_USER, LTC_PASSWORD)
+        if token:
+            s.set("token", token, expire=3600*24*1) #  pēc 1d ekspirejas
+            print "** %s: token=%s" % (request.remote_addr,token)
+        else:
+            print "Can not login"
+            raise bottle.HTTPError(500, "Can not login")
+
+# @app.route('/playstream/<url:re:.*>')
+
+
+### Live playlist ###
+@app.route("/<key>/live/<ch>/")
+def get_live(key, ch):
+    global s, token, headers0
+    path0, rest = hls_split(request.url)
+    response.content_type = "application/x-mpegURL" # r.headers["content-type"] # 'application/vnd.apple.mpegurl' # application/x-mpegURL
+    if "c"+ch in s:
+        stream_url2 = s["c"+ch]
+        mediaid = s["m"+ch]
+        print "** %s: serving live playlist for %s (%s) from cache" % (request.remote_addr,path0,mediaid )
+    else:
+        stream_url2, mediaid = refresh_live_chunklist_url(ch)
+        print "** %s: getting ive playlist for %s (%s)" % (request.remote_addr,path0,mediaid )
+    stream_url2 += token
+
+    if REDIRECT:
+        bottle.redirect(stream_url2, 307)
+
+    for i in range(3):
+        r2 = requests.get(stream_url2,headers=headers0)
+        if r2.status_code == 200:
+            break
+        time.sleep(1)
+    else:
+        print "Error %s getting live chunklist %s"% (r2.status_code,stream_url2)
+        raise bottle.HTTPError(r2.status_code)
+    return r2.content
+
+### Live TS chunk ###
+@app.route("/<key>/live/<ch>/<tail>")
+def get_live_chunk(key, ch, tail):
+    global s, token, headers0
+    path0, rest = hls_split(request.url)
+    chid = re.search("resource_id=c-(\\w+)",rest).group(1)
+    chunkid = re.search("(\d+)\.ts", request.url).group(1)
+    path2 = ch + "/" +  chunkid
+    if CACHE and path2 in s:
+        print "** %s: serving live ts %s from cache" % (request.remote_addr,path2)
+        f = s.get(path2, read=True)
+        response.headers["Content-Type"] =  s[path2+"@"] #'video/MP2T'
+        response.headers["Content-Length"] = s[path2+"#"]
+        while True:
+            chunk = f.read(8192)
+            if not chunk:
+                break
+            yield chunk
+
+    else: #  no cache
+        if ch in s:
+            stream_url = s[ch]
+            mediaid= s["m"+ch]
+        else:
+            print "No stream_url %s in cache" % path0
+            raise bottle.HTTPError(500)
+        base0, rest0 = hls_base(stream_url)
+        rest2 = "media_%s_%s.ts?resource_id=c-%s&auth_token=app_" % (mediaid, chunkid, chid)
+        url = base0 + rest2 + token
+        url2 = hls_base(stream_url)[0] + rest
+        headers = dict(request.headers)
+        del headers["Host"]
+        # headers["Authorization"] = "Bearer " + token
+        print "** %s: getting live ts from %s(%s)- %s" % (request.remote_addr, path2, mediaid,url[:40])
+        if DEBUG:
+            print "=== Request headers ==="
+            print_headers(headers)
+        r = requests.get(url, stream=True, headers=headers0)
+        if r.status_code <> 200:
+            r = requests.get(url, stream=True, headers=headers0) # try once more
+            if r.status_code <> 200:
+                # Refresh chunklist
+                print "## %s: Refreshing chunklist/mediaid  for live channel %s" %(request.remote_addr, ch)
+                chunklist_url, mediaid = refresh_live_chunklist_url(ch)
+                rest2 = "media_%s_%s.ts?resource_id=c-%s&auth_token=app_" % (mediaid, chunkid, chid)
+                url = base0 + rest2 + token
+                url2 = chunklist_url + token
+                print "** %s: getting live ts from %s(%s)- %s" % (request.remote_addr, path2, mediaid,url[:40])
+                r = requests.get(url, stream=True, headers=headers0)
+                if r.status_code <> 200:
+                    print "Error %s opening stream \n%s" %(r.status_code,url)
+                    print url2
+                    raise bottle.HTTPError(r.status_code, "Error opening stream "+url)
+
+        content = ""
+        response.content_type = r.headers["content-type"] # 'application/vnd.apple.mpegurl' #
+        # response.headers.clear()
+        for k in r.headers:
+            response.headers[k] =  r.headers[k]
+        if DEBUG:
+            print "=== Response headers ==="
+            print_headers(response.headers)
+        for chunk in r.iter_content(chunk_size=8192):
+            if chunk:
+                content += chunk
+                yield chunk
+        if len(content) <> int(r.headers["content-length"]):
+            print "Content length problem"
+        if CACHE:
+            s.set(path2, content, expire=3600, read=True)
+            s.set(path2+"#", len(content), expire=3600, read=True)
+            s.set(path2+"@", r.headers["Content-Type"], expire=3600, read=True)
+
+### Archive playlist ###
+@app.route("/<key>/live/<ch>/<ts>/")
+def get_archive(key, ch, ts):
+    global s, token, headers0
+    path0, rest = hls_split(request.url)
+    start = int(ts) + 60 * 5
+    epg = get_epg(ch, start)
+    print "** %s: getting archive playlist for channel %s" % (request.remote_addr,path0)
+    if epg:
+        epgid = epg["id"]
+        epg_start = int(epg["attributes"]["unix-start"])
+        epg_stop = int(epg["attributes"]["unix-stop"])
+        epg_title = epg["attributes"]["title"]
+    else:
+        print "EPG not found"
+        raise bottle.HTTPError(500, "EPG not found")
+
+    stream_url = epg_get_stream_url(epgid)
+    if REDIRECT:
+        bottle.redirect(stream_url, 307)
+
+    # Getting chunklist
+    stream_url2, mediaid = refresh_epg_chunklist_url(stream_url)
+    r2 = requests.get(stream_url2)
+    if r2.status_code <> 200:
+        print "Error %s getting archive chunklist %s"% (r2.status_code,stream_url2)
+        raise bottle.HTTPError(r2.status_code)
+    result = re.findall(r"#EXTINF:([\d\.]+),\n(.+)", r2.content)
+    ll = 0
+    i = 0
+    for chunk_len, chunk_url in result:
+        ll += float(chunk_len)
+        if ll > (start - epg_start):
+            break
+        i += 1
+    result2 =result[i:]
+    content = re.search("(^.+?)#EXTINF", r2.content, re.DOTALL).group(1)
+    for chunk_len, chunk_url in result2:
+        content += "#EXTINF:%s,\n" % chunk_len
+        content += chunk_url + "\n"
+    content += "#EXT-X-ENDLIST"
+    response.content_type = r2.headers["content-type"] # 'application/vnd.apple.mpegurl' #
+    return content
+
+def live_get_stream_url(ch):
+    global s, token, headers0
+    if ch in s:
+        stream_url = s[ch]
+        stream_url += token
+    else:
+        # Getting live stream url
+        url = url0 + "live-streams/%s?include=quality&auth_token=app_%s" % (ch, token)
+        headers = headers0.copy()
+        headers["Authorization"] = "Bearer " + token
+        r = requests.get(url, headers=headers)
+        if r.status_code <> 200:
+            print "Error getting epg stream url "+url
+            raise bottle.HTTPError(r.status_code, "Error getting epg stream url "+url)
+        js = json.loads(r.content)
+        stream_url = js["data"][0]["attributes"]["stream-url"]
+        stream_url0 = stream_url.replace(token, "")
+        s.set(ch, stream_url0, expire=3600*24*7, read=False)
+    return str(stream_url)
+
+
+def epg_get_stream_url(epgid):
+    global s, token, headers0
+    if epgid in s:
+        stream_url = s[epgid]
+        stream_url += token
+    else:
+        # Getting epg stream url
+        url = url0 + "record-streams/%s?include=quality&auth_token=app_%s" % (epgid, token)
+        headers = headers0.copy()
+        headers["Authorization"] = "Bearer " + token
+        r = requests.get(url, headers=headers)
+        if r.status_code <> 200:
+            print "Error getting epg stream url "+url
+            raise bottle.HTTPError(r.status_code, "Error getting epg stream url "+url)
+        js = json.loads(r.content)
+        stream_url = js["data"][0]["attributes"]["stream-url"]
+        stream_url0 = stream_url.replace(token, "")
+        s.set(epgid, stream_url0, expire=3600*24*7, read=False)
+    return str(stream_url)
+
+def refresh_live_chunklist_url(ch):
+    global s, token, headers0
+    stream_url = live_get_stream_url(ch)
+    r = requests.get(stream_url)
+    if r.status_code <> 200:
+        print "Error %s getting live chunklist %s"% (r.status_code,stream_url)
+        raise bottle.HTTPError(r.status_code)
+    chid = re.search("resource_id=c\\-(\\w+)",stream_url).group(1)
+    rest2 = re.search("chunklist.+$", r.content).group(0).replace(token,"")
+    mediaid = re.search("chunklist_(.+?)\\.m3u8",rest2).group(1)
+    base2 = hls_base(stream_url)[0]
+    stream_url2 = base2 + rest2 # chunlist url
+    s.set("m"+ch, mediaid, expire=3600*24*7, read=False)
+    s.set("c"+ch, stream_url2, expire=3600*24*7, read=False)
+    return stream_url2,mediaid
+
+def refresh_epg_chunklist_url(stream_url):
+    global s, token, headers0
+    r = requests.get(stream_url)
+    if r.status_code <> 200:
+        print "Error %s getting archive chunklist %s"% (r.status_code,stream_url)
+        raise bottle.HTTPError(r.status_code)
+    epgid = re.search("resource_id=a-(\\d+)",stream_url).group(1)
+    rest2 = re.search("chunklist.+$", r.content).group(0)
+    mediaid = re.search("chunklist_(.+?)\\.m3u8",rest2).group(1)
+    s.set("m"+epgid, mediaid, expire=3600*24*7, read=False)
+    base2 = hls_base(stream_url)[0]
+    stream_url2 = base2 + rest2 # chunlist url
+    return stream_url2,mediaid
+
+
+### Archive ts chunk ###
+@app.route("/<key>/live/<ch>/<ts>/<tail>")
+def get_archive_chunk(key, ch, ts, tail):
+    global s, token, headers0
+    path0, rest = hls_split(request.url)
+    epgid = re.search("resource_id=a-(\\d+)",rest).group(1)
+    chunkid = re.search("(\\d+)\\.ts", rest).group(1)
+    path2 = epgid + "/" +  chunkid
+    if CACHE and path2 in s:
+        print "** %s: serving archive ts from cache %s" % (request.remote_addr,path2)
+        f = s.get(path2, read=True)
+        response.headers["Content-Type"] =  s[path2+"@"] #'video/MP2T'
+        response.headers["Content-Length"] = s[path2+"#"]
+        while True:
+            chunk = f.read(8192)
+            if not chunk:
+                break
+            yield chunk
+
+    else: #  No cache
+        stream_url = epg_get_stream_url(epgid)
+        if "m"+epgid in s:
+            mediaid= s["m"+epgid]
+        else:
+            chunklist_url, mediaid = refresh_epg_chunklist_url(stream_url)
+        base0, rest0 = hls_base(stream_url)
+        #media_w76603200_0.ts?resource_id=a-6559656352477&auth_token=app_
+        rest2 = "media_%s_%s.ts?resource_id=a-%s&auth_token=app_" % (mediaid, chunkid, epgid)
+        url = base0 + rest2 + token
+        print "** %s: getting archive ts from %s(%s) - %s" % (request.remote_addr,path2, mediaid, rest2[:rest2.index("?")])
+        #print url
+        headers = dict(request.headers)
+        del headers["Host"]
+        # headers["Authorization"] = "Bearer " + token
+
+        r = requests.get(url, stream=True, headers=headers)
+        if r.status_code <> 200:
+            r = requests.get(url, stream=True, headers=headers) # try once more
+            if r.status_code <> 200:
+                # Refresh chunklist
+                print "## %s: Refreshing chunklist/mediaid  for epg %s" %(request.remote_addr, epgid)
+                chunklist_url, mediaid = refresh_epg_chunklist_url(stream_url)
+                rest2 = "media_%s_%s.ts?resource_id=a-%s&auth_token=app_" % (mediaid, chunkid, epgid)
+                url = base0 + rest2 + token
+                print "** %s: getting archive ts from %s(%s) - %s" % (request.remote_addr, path2, mediaid, rest2[:rest2.index("?")])
+                r = requests.get(url, stream=True, headers=headers0)
+                if r.status_code <> 200:
+                    print "Error %s opening stream \n%s" %(r.status_code,url)
+                    raise bottle.HTTPError(r.status_code, "Error opening stream "+url)
+
+        content = ""
+        response.content_type = r.headers["content-type"] # 'application/vnd.apple.mpegurl' #
+        # response.headers.clear()
+        for k in r.headers:
+            response.headers[k] =  r.headers[k]
+        if DEBUG:
+            print_headers(response.headers)
+        for chunk in r.iter_content(chunk_size=8192):
+            if chunk:
+                content += chunk
+            yield chunk
+        if CACHE:
+            path2 = epgid + "/" + chunkid
+            s.set(path2, content, expire=3600, read=True)
+            s.set(path2+"#", len(content), expire=3600, read=True)
+            s.set(path2+"@", r.headers["Content-Type"], expire=3600, read=True)
+
+
+@app.route("/<key>/vod/<ch>/")
+def get_vod(key, ch):
+    global s, token, headers0
+    path0, rest = hls_split(request.url)
+    if path0 in s:
+        stream_url = s[path0] + token
+        print "** %s: getting vod to %s from cache (%s)" % (request.remote_addr, path0)
+    else:
+        url = url0 + "vod-streams/%s?include=language,subtitles,quality &auth_token=app_%s" % (ch, token)
+        headers = headers0.copy()
+        headers["Authorization"] = "Bearer " + token
+        r = requests.get(url, headers=headers)
+        if r.status_code <> 200:
+            raise bottle.HTTPError(r.status_code, "Error opening stream "+url)
+        js = json.loads(r.content)
+        stream_url = js["data"][0]["attributes"]["stream-url"]
+        stream_url0 = stream_url.replace(token, "")
+        s.set(path0, stream_url0, expire=3600*24*7, read=False)
+        print "** %s: changing vod to %s (%s)" % (request.remote_addr, path0)
+    if True: # REDIRECT:
+        bottle.redirect(stream_url, 307)
+    r = requests.get(stream_url)
+    if r.status_code <> 200:
+        raise bottle.HTTPError(r.status_code)
+    response.content_type = r.headers["content-type"] # 'application/vnd.apple.mpegurl' #
+    return r.content
+
+
+def get_epg(ch, start):
+    url = url0 + "epgs/?filter[channel]=%s&filter[utFrom]=%s&filter[utTo]=%s&include=channel&page[size]=40page[number]=1" % (ch, start, start )
+    r = requests.get(url)
+    if r.status_code <> 200:
+        raise bottle.HTTPError(500, "EPG not found")
+    js = json.loads(r.content)
+    if "data" in js:
+        for epg in js["data"]:
+            if int(epg["id"]) < 0:
+                continue
+            else:
+                break
+        return epg
+    else:
+        return None
+
+
+####################################################################
+# Run WSGI server
+def start(server,port):
+    print "*** Starting ltcproxy ***"
+    options = {}
+    if server == "mtwsgi":
+        import mtwsgi
+        server = mtwsgi.MTServer
+        options = {"thread_count": WORKERS,}
+
+    run(app=app,server=server, host='0.0.0.0',
+            port=port,
+            reloader=False,
+            quiet=False,
+            plugins=None,
+            debug=True,
+            config=None,
+            **options)
+
+def login(user,password):
+    """Login in to site, get token"""
+
+    # Dabūjam tokenu
+    url = "https://manstv.lattelecom.tv/api/v1.7/post/user/users/%s" % user
+    params = "uid=5136baee57505694&password=%s&" % (password)
+    headers = headers2dict("""
+User-Agent: Shortcut.lv v2.9.1 / Dalvik/1.6.0 (Linux; U; Android 4.4.2; SM-G900FD Build/KOT49H)
+Content-Type: application/x-www-form-urlencoded; charset=UTF-8
+Host: manstv.lattelecom.tv
+""" )
+    try:
+        r = urllib2.Request(url, data=params, headers=headers)
+        u = urllib2.urlopen(r)
+        content = u.read()
+        u.close()
+    except Exception as ex:
+        return None
+    if r and "token" in content:
+        token = re.search('"token":"(.+?)"', content).group(1)
+        return token
+    else:
+        return False
+
+def refresh_token(token):
+    """Refresh"""
+
+    url = "https://manstv.lattelecom.tv/api/v1.7/post/user/refresh-token/"
+    params = "uid=5136baee57505694&token=%s&" % (token)
+    headers = headers2dict("""
+User-Agent: Shortcut.lv v2.9.1 / Dalvik/1.6.0 (Linux; U; Android 4.4.2; SM-G900FD Build/KOT49H)
+Content-Type: application/x-www-form-urlencoded; charset=UTF-8
+Host: manstv.lattelecom.tv
+""" )
+    try:
+        r = urllib2.Request(url, data=params, headers=headers)
+        u = urllib2.urlopen(r)
+        content = u.read()
+        u.close()
+    except Exception as ex:
+        return None
+    if r and "token" in content:
+        token2 = re.search('"token":"(.+?)"', content).group(1)
+        return token2
+    else:
+        return False
+
+def print_headers(headers):
+    for h in headers:
+        print "%s: %s"%(h,headers[h])
+
+def del_headers(headers0,tags):
+    headers = headers0.copy()
+    for t in tags:
+        if t in headers:
+            del headers[t]
+        if t.lower() in headers:
+            del headers[t.lower()]
+    return headers
+
+def hls_split(url):
+    pp = urlparse.urlsplit(url)
+    path0 = pp.path[:pp.path.rindex("/")+1]
+    path0 = path0[path0.index("/", 1):]
+    rest = pp.path[pp.path.rindex("/")+1:] + "?" + pp.query
+    return path0, rest
+
+def hls_base(url):
+    base = url.split("?")[0]
+    base = "/".join(base.split("/")[0:-1])+ "/"
+    rest = url.replace(base, "")
+    return base, rest
+
+#########################################################################################
+if __name__ == '__main__':
+    # 1561839586
+    # get_epg("101", 1561839586)
+
+    try:
+        opts, args = getopt.gnu_getopt(sys.argv[1:], "p:s:dr", ["port=","server=","--debug"])
+    except getopt.GetoptError as err:
+        print str(err)
+        print str(__doc__)
+        sys.exit(2)
+    opts = dict(opts)
+
+    if not len(args):
+        print str(__doc__)
+        sys.exit(2)
+
+    if "-r" in opts:
+        print "Enabling remote debuging (ptvsd)"
+        import ptvsd
+        ptvsd.enable_attach(address = ('0.0.0.0', 5678),redirect_output=False)
+    if "-d" in opts:
+        print "Enabling debuging mode (more output)"
+        DEBUG = True
+    pid = "/var/run/ltcproxy.pid"
+    daemon = daemonize.Daemon(start, pid)
+    server =  opts["-s"] if "-s" in opts else SERVER
+    port = opts["-p"] if "-p" in opts else PORT_NUMBER
+
+    if "start" == args[0]:
+        s.clear()
+        daemon.start(server,port)
+        daemon.is_running()
+    elif "stop" == args[0]:
+        daemon.stop()
+    elif "restart" == args[0]:
+        s.clear()
+        daemon.restart()
+        daemon.is_running()
+    elif "manualstart" == args[0]:
+        s.clear()
+        start(server,port)
+    elif "status" == args[0]:
+        daemon.is_running()
+    else:
+        print "Unknown command"
+        print str(__doc__)
+        sys.exit(2)
+    sys.exit(0)