content/util.py

# -*- coding: UTF-8 -*-
# /*
# *      Copyright (C) 2011 Libor Zoubek,ivars777
# *
# *
# *  This Program is free software; you can redistribute it and/or modify
# *  it under the terms of the GNU General Public License as published by
# *  the Free Software Foundation; either version 2, or (at your option)
# *  any later version.
# *
# *  This Program is distributed in the hope that it will be useful,
# *  but WITHOUT ANY WARRANTY; without even the implied warranty of
# *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# *  GNU General Public License for more details.
# *
# *  You should have received a copy of the GNU General Public License
# *  along with this program; see the file COPYING.  If not, write to
# *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
# *  http://www.gnu.org/copyleft/gpl.html
# *
# */
import os, sys, re
import urllib, urllib2
import datetime, time
import traceback
import cookielib
import requests
try:
    from requests.packages.urllib3.exceptions import InsecureRequestWarning
    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
except:
    pass
from htmlentitydefs import name2codepoint as n2cp
import HTMLParser
import StringIO

#import threading
#import Queue
import pickle
import string
import simplejson as json
#from demjson import demjson
#import demjson
import json
#from bs4 import BeautifulSoup

UA = 'Mozilla/6.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.5) Gecko/2008092417 Firefox/3.0.3'
LOG = 2

_cookie_jar = None
CACHE_COOKIES = 'cookies'

def system():
    if "kodi" in sys.executable.lower():
        return "kodi"
    elif sys.platform == "win32":
        return "windows"
    elif sys.platform == "linux2":
        return "enigma2"
    else:
        return "unknown"

def nfo2xml(nfo_dict):
    nfo_type,nfo = next(nfo_dict.iteritems())
    s= "<%s>\n"%nfo_type.encode("utf8")
    for k,v in nfo.iteritems():
        if isinstance(v,list):
            for v2 in v:
                if isinstance(v2,unicode): v2 = v2.encode("utf8")
                s += "    <%s>%s</%s>\n"%(k.encode("utf8"), v2, k.encode("utf8"))
        else:
            if isinstance(v,unicode): v = v.encode("utf8")
            s += "    <%s>%s</%s>\n"%(k.encode("utf8"), v, k.encode("utf8"))
    s += "</%s>\n"%nfo_type.encode("utf8")
    return s

def nfo2desc(nfo):

    if not "title" in nfo:
        nfo_type, nfo = next(nfo.iteritems())
    desc = nfo2title(nfo)
    dd = lambda t: "\n" + nfo[t] if t in nfo and nfo[t] else ""
    dd2 = lambda t: "\n" + ",".join(nfo[t]) if t in nfo and nfo[t] else ""

    def ddd(t,title=""):
        if title:
            title = title + ": "
        if t in nfo and nfo[t]:
            if isinstance(nfo[t],list):
                d = "\n" + title + ",".join(nfo[t])
            else:
                d = "\n" + title + nfo[t]
        else:
            d = ""
        return d.encode("utf8") if isinstance(d, unicode) else d

    desc += ddd("tagline")
    if "plot" in nfo and "tagline" in nfo and nfo["tagline"] <> nfo["plot"]:
        desc += ddd("plot")
    elif "plot" in nfo and not "tagline" in nfo:
        desc += ddd("plot")
    desc += ddd("genre","Genre")
    desc += ddd("runtime","Length")
    desc += ddd("director","Director")
    desc += ddd("actor","Actors")
    desc += ddd("language","Languages")
    desc += ddd("quality","Quality")
    return desc.encode("utf8") if isinstance(desc,unicode) else desc

def nfo2title(nfo):
    if not "title" in nfo:
        nfo_type, nfo = next(nfo.iteritems())
    title = nfo["title"]
    if "originaltitle" in nfo and nfo["originaltitle"] and nfo["originaltitle"]<>nfo["title"]:
        title +=" ~ "+nfo["originaltitle"]
    if "year" in nfo and nfo["year"]:
        title += " (%s)"%nfo["year"]
    return title.encode("utf8") if isinstance(title,unicode) else title

def stream_type(data):
    data = data.lower()
    m = re.search(r"^(\w+)://", data)
    prefix = m.group(1) if m else ""
    if prefix in ("http","https"):
        if ".m3u8" in data:
            return "hls"
        elif ".mpd" in data:
            return "dash"
        else:
            return "http"
    else:
        return prefix

def check_version(package,url="http://feed.blue.lv/Packages"):
    "Return current package version from OPKG feed"
    url = "http://feed.blue.lv/Packages"
    r = requests.get(url)
    if not r.ok:
        return ""
    m = re.search("Package: %s\nVersion: (.+?)\n"%package, r.content)
    if not m:
        return ""
    return m.group(1)

SPLIT_CHAR = "~"
SPLIT_CODE = urllib.quote(SPLIT_CHAR)
EQ_CODE = urllib.quote("=")
COL_CODE = urllib.quote(":")
SPACE_CODE = urllib.quote(" ")

def make_fname(title):
    "Make file name from title"
    title = title.strip()
    fname0 = re.sub("[/\n\r\t,:]"," ",title)
    fname0 = re.sub("['""]","",fname0)
    return fname0

def hls_base(url):
    url2 = url.split("?")[0]
    url2 = "/".join(url2.split("/")[0:-1])+ "/"
    return url2

def stream_change(stream):
    #return stream # TODO
    if "resolver" in stream and stream["resolver"] in ("viaplay","hqq","filmas") or \
        "surl" in stream and re.search("https*://(hqq|goo\.gl)",stream["surl"]):
        stream["url"] = streamproxy_encode(stream["url"],stream["headers"])
        stream["headers"] = {}
        return stream
    else:
        return stream

def streamproxy_encode(url,headers=[],proxy_url=None):
    PROXY_URL = "http://localhost:8880/"
    if not "?" in url:
        url = url+"?"
    url2 = url.replace(SPLIT_CHAR,SPLIT_CODE).replace(":",COL_CODE).replace(" ",SPACE_CODE)
    if not proxy_url:
        proxy_url = PROXY_URL
    url2 = proxy_url + url2
    if headers:
        headers2 = []
        for h in headers:
            headers2.append("%s=%s"%(h,headers[h].replace("=",EQ_CODE).replace(SPLIT_CHAR,SPLIT_CODE).replace(" ",SPACE_CODE)))
        headers2 = SPLIT_CHAR.join(headers2)
        url2 = url2+SPLIT_CHAR+headers2
    #return url2.encode("utf8") if isinstance(url2,unicode) else url2
    return url2

def streamproxy_decode(urlp):
    import urlparse
    path = urlp.replace(re.search("http://[^/]+",urlp).group(0),"")
    p = path.split(SPLIT_CHAR)
    url = urllib.unquote(p[0][1:])
    #headers = {"User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 9_2 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/47.0.2526.70 Mobile/13C71 Safari/601.1.46"}
    headers={}
    if len(p)>1:
        for h in p[1:]:
            #h = urllib.unquote()
            headers[h.split("=")[0]]=urllib.unquote(h.split("=")[1])
    return url,headers

def streamproxy_encode2(url,headers=[],proxy_url=None):
    PROXY_URL = "http://localhost:8880/"
    #url2 = url.replace(SPLIT_CHAR,SPLIT_CODE).replace(":",COL_CODE).replace(" ",SPACE_CODE)
    url2 = urllib.quote_plus(url)
    if not proxy_url:
        proxy_url = PROXY_URL
    url2 = proxy_url + url2+"/?"
    if headers:
        headers2 = []
        for h in headers:
            headers2.append("%s=%s"%(h,headers[h].replace("=",EQ_CODE).replace(SPLIT_CHAR,SPLIT_CODE).replace(" ",SPACE_CODE)))
        headers2 = SPLIT_CHAR.join(headers2)
        url2 = url2+SPLIT_CHAR+headers2
    return url2

def streamproxy_decode2(urlp):
    path = urlp.replace(re.search("http://[^/]+",urlp).group(0),"")
    p = path.split(SPLIT_CHAR)
    url = urllib.unquote_plus(p[0][1:-2])
    #headers = {"User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 9_2 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/47.0.2526.70 Mobile/13C71 Safari/601.1.46"}
    headers={}
    if len(p)>1:
        for h in p[1:]:
            #h = urllib.unquote()
            headers[h.split("=")[0]]=urllib.unquote(h.split("=")[1])
    return url,headers

class Captions(object):
    def __init__(self,uri):
        self.uri = uri
        self.subs = []
        self.styles = {}
        if uri.startswith("http"):
            r = requests.get(uri)
        if r.status_code == 200:
            self.loads(r.content)

    def loads(self,s):
        if "WEBVTT" in s[:s.find("\n")]: # vtt captions
            self.load_vtt(s)
        elif "<?xml" in s[:s.find("\n")]:
            self.load_ttaf(s)
        else:
            self.load_vtt(s) # TODO


    def load_ttaf(self,s):
        for r2 in re.findall("<style .+?/>", s):
            st = {}
            for a in re.findall(r'(\w+)="([^ "]+)"', r2):
                st[a[0]] = a[1]
                if a[0] == "id":
                    sid = a[1]
            self.styles[sid] = st
        for r2 in re.findall("<p .+?</p>", s):
            sub = {}
            sub["begin"] = str2sec(re.search('begin="([^"]+)"', r2).group(1)) if re.search('begin="([^"]+)"', r2) else -1
            sub["end"] = str2sec(re.search('end="([^"]+)"', r2).group(1)) if re.search('end="([^"]+)"', r2) else -1
            sub["style"] = re.search('style="([^"]+)"', r2).group(1) if re.search('style="([^"]+)"', r2) else None
            sub["text"] = re.search("<p[^>]+>(.+)</p>", r2).group(1).replace("\n","")
            sub["text"] = re.sub("<br\s*?/>","\n",sub["text"])
            sub["text"] = re.sub("<.+?>"," ",sub["text"])
            self.subs.append(sub)
        pass

    def load_vtt(self,s):
        f = StringIO.StringIO(s)
        while True:
            line = f.readline()
            if not line:
                break
            m = re.search(r"([\d\.\,:]+)\s*-->\s*([\d\.\,\:]+)",line)
            if m:
                sub = {}
                sub["begin"] = str2sec(m.group(1))
                sub["end"] = str2sec(m.group(2))
                sub["style"] = None
                sub["text"] = []
                line = f.readline()
                while line.strip():
                    txt = line.strip()
                    if isinstance(txt,unicode):
                        txt = txt.encode("utf8")
                    sub["text"].append(txt)
                    line = f.readline()
                sub["text"] = "\n".join(sub["text"])
                self.subs.append(sub)
            else:
                continue
        pass

    def get_srt(self):
        out = ""
        i = 0
        for sub in self.subs:
            i +=1
            begin = sub["begin"]
            begin = "%s,%03i"%(str(datetime.timedelta(seconds=begin/1000)),begin%1000)
            end = sub["end"]
            end = "%s,%03i"%(str(datetime.timedelta(seconds=end/1000)),end%1000)
            txt2 = sub["text"]
            out += "%s\n%s --> %s\n%s\n\n\n"%(i,begin,end,txt2)
        return out

def str2sec(r):
    # Convert str time to miliseconds
    r= r.replace(",",".")
    m = re.search(r"(\d+\:)*(\d+)\:(\d+\.\d+)", r)
    if m:
        sec = int(m.group(1)[:-1])*60*60*1000 if m.group(1) else 0
        sec += int(m.group(2))*60*1000 + int(float(m.group(3))*1000)
        return sec
    else:
        return -1


#c = Captions("http://195.13.216.2/mobile-vod/mp4:lb_barbecue_fr_lq.mp4/lb_barbecue_lv.vtt")
#c = Captions("http://www.bbc.co.uk/iplayer/subtitles/ng/modav/bUnknown-0edd6227-0f38-411c-8d46-fa033c4c61c1_b05ql1s3_1479853893356.xml")
#url = "http://195.13.216.2/mobile-vod/mp4:ac_now_you_see_me_2_en_lq.mp4/ac_now_you_see_me_2_lv.vtt"
#c = Captions(url)

#pass


def ttaf2srt(s):
    out = u""
    i = 0
    for p,txt in re.findall("<p ([^>]+)>(.+?)</p>", s, re.DOTALL):
        i +=1
        begin = re.search('begin="(.+?)"',p).group(1)
        begin = begin.replace(".",",")
        end = re.search('end="(.+?)"',p).group(1)
        end = end.replace(".",",")
        txt2 = re.sub("<br */>","\n",txt)
        out += "%s\n%s --> %s\n%s\n\n"%(i,begin,end,txt2)
    return out

def vtt2srt(s):
    out = ""
    i = 0
    # result = re.findall(r"([\d:\. ]+-->[\d:\. ]+)\n(.+?\n)+", r.content)
    for t1, t2, txt in re.findall(r"([\d:\.]+)\s*-->\s*([\d:\.]+)\n(.+?\n)+", s):
        i +=1
        if len(t1.split(":")) == 2:
            t1 = "00:" + t1
        t1 = t1.replace(".", ",")
        if len(t2.split(":")) == 2:
            t2 = "00:" + t2
        t2 = t2.replace(".", ",")
        out += "%s\n%s --> %s\n%s\n\n"%(i,t1,t2,txt)
    return out


def item():
    """Default item content"""
    stream0 = {
        'name': '', #
        'url': '',
        'quality': '?',
        'surl': '',
        'subs': [],
        'headers': {},
        "desc":"",
        "img":"",
        "lang":"",
        "type":"",
        "resolver":"",
        "order":0,
        "live":False
        }
    return stream0

class _StringCookieJar(cookielib.LWPCookieJar):

    def __init__(self, string=None, filename=None, delayload=False, policy=None):
        cookielib.LWPCookieJar.__init__(self, filename, delayload, policy)
        if string and len(string) > 0:
            self._cookies = pickle.loads(str(string))

    def dump(self):
        return pickle.dumps(self._cookies)


def init_urllib(cache=None):
    """
    Initializes urllib cookie handler
    """
    global _cookie_jar
    data = None
    if cache is not None:
        data = cache.get(CACHE_COOKIES)
    _cookie_jar = _StringCookieJar(data)
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(_cookie_jar))
    urllib2.install_opener(opener)


def cache_cookies(cache):
    """
    Saves cookies to cache
    """
    global _cookie_jar
    if _cookie_jar:
        cache.set(CACHE_COOKIES, _cookie_jar.dump())


def request0(url, headers={}):
    debug('request: %s' % url)
    req = urllib2.Request(url, headers=headers)
    req.add_header('User-Agent', UA)
    try:
        response = urllib2.urlopen(req)
        data = response.read()
        response.close()
    except urllib2.HTTPError, error:
        data = error.read()

    debug('len(data) %s' % len(data))
    return data

def request(url, headers={}):
    debug('request: %s' % url)
    #req = urllib2.Request(url, headers=headers)
    #req.add_header('User-Agent', UA)
    if 'User-Agent' not in headers:
        headers['User-Agent']= UA
    try:
        r = requests.get(url, headers=headers)
        data = r.content
    except:
        data = r.content

    debug('len(data) %s' % len(data))
    return data

def post(url, data, headers={}):
    postdata = urllib.urlencode(data)
    #req = urllib2.Request(url, postdata, headers)
    #req.add_header('User-Agent', UA)
    import requests
    if 'User-Agent' not in headers:
        headers['User-Agent']= UA
    try:
        r = requests.post(url, data=postdata,headers=headers)
        data = r.content
    except urllib2.HTTPError, error:
        data = r.content
    return data

def post0(url, data, headers={}):
    postdata = urllib.urlencode(data)
    req = urllib2.Request(url, postdata, headers)
    req.add_header('User-Agent', UA)
    try:
        response = urllib2.urlopen(req)
        data = response.read()
        response.close()
    except urllib2.HTTPError, error:
        data = error.read()
    return data


def post_json(url, data, headers={}):
    postdata = json.dumps(data)
    headers['Content-Type'] = 'application/json'
    req = urllib2.Request(url, postdata, headers)
    req.add_header('User-Agent', UA)
    response = urllib2.urlopen(req)
    data = response.read()
    response.close()
    return data


#def run_parallel_in_threads(target, args_list):
    #result = Queue.Queue()
    ## wrapper to collect return value in a Queue

    #def task_wrapper(*args):
        #result.put(target(*args))

    #threads = [threading.Thread(target=task_wrapper, args=args) for args in args_list]
    #for t in threads:
        #t.start()
    #for t in threads:
        #t.join()
    #return result


def substr(data, start, end):
    i1 = data.find(start)
    i2 = data.find(end, i1)
    return data[i1:i2]


def save_to_file(url, file):
    try:
        return save_data_to_file(request(url), file)
    except:
        traceback.print_exc()


def save_data_to_file(data, file):
    try:
        f = open(file, 'wb')
        f.write(data)
        f.close()
        info('File %s saved' % file)
        return True
    except:
        traceback.print_exc()


def read_file(file):
    if not os.path.exists(file):
        return ''
    f = open(file, 'r')
    data = f.read()
    f.close()
    return data


def _substitute_entity(match):
    ent = match.group(3)
    if match.group(1) == '#':
        # decoding by number
        if match.group(2) == '':
            # number is in decimal
            return unichr(int(ent))
        elif match.group(2) == 'x':
            # number is in hex
            return unichr(int('0x' + ent, 16))
    else:
        # they were using a name
        cp = n2cp.get(ent)
        if cp:
            return unichr(cp)
        else:
            return match.group()


def decode_html(data):
    if not type(data) == str:
        return data
    try:
        if not type(data) == unicode:
            data = unicode(data, 'utf-8', errors='ignore')
        entity_re = re.compile(r'&(#?)(x?)(\w+);')
        return entity_re.subn(_substitute_entity, data)[0]
    except:
        traceback.print_exc()
        print[data]
        return data

def unescape(s0):
    #s2 = re.sub("&#\w+;",HTMLParser.HTMLParser().unescape("\1"),s)
    s0 = s0.replace("&amp;","&").replace("&quot;", '"')
    for s in re.findall("&#\w+;",s0):
        s2 = HTMLParser.HTMLParser().unescape(s)
        if isinstance(s0,str):
            s2 = s2.encode("utf8")
        s0 = s0.replace(s,s2)
        pass
    return s0

def debug(text):
    if LOG > 1:
        print('[DEBUG] ' + str([text]))

def info(text):
    if LOG > 0:
        print('[INFO] ' + str([text]))

def error(text):
    print('[ERROR] ' + str([text]))

_diacritic_replace = {u'\u00f3': 'o',
                      u'\u0213': '-',
                      u'\u00e1': 'a',
                      u'\u010d': 'c',
                      u'\u010c': 'C',
                      u'\u010f': 'd',
                      u'\u010e': 'D',
                      u'\u00e9': 'e',
                      u'\u011b': 'e',
                      u'\u00ed': 'i',
                      u'\u0148': 'n',
                      u'\u0159': 'r',
                      u'\u0161': 's',
                      u'\u0165': 't',
                      u'\u016f': 'u',
                      u'\u00fd': 'y',
                      u'\u017e': 'z',
                      u'\xed': 'i',
                      u'\xe9': 'e',
                      u'\xe1': 'a',
                      }


def replace_diacritic(string):
    ret = []
    for char in string:
        if char in _diacritic_replace:
            ret.append(_diacritic_replace[char])
        else:
            ret.append(char)
    return ''.join(ret)


def params(url=None):
    if not url:
        url = sys.argv[2]
    param = {}
    paramstring = url
    if len(paramstring) >= 2:
        params = url
        cleanedparams = params.replace('?', '')
        if (params[len(params) - 1] == '/'):
            params = params[0:len(params) - 2]
        pairsofparams = cleanedparams.split('&')
        param = {}
        for i in range(len(pairsofparams)):
            splitparams = {}
            splitparams = pairsofparams[i].split('=')
            if (len(splitparams)) == 2:
                param[splitparams[0]] = splitparams[1]
    for p in param.keys():
        param[p] = param[p].decode('hex')
    return param


def int_to_base(number, base):
    digs = string.digits + string.letters
    if number < 0:
        sign = -1
    elif number == 0:
        return digs[0]
    else:
        sign = 1
    number *= sign
    digits = []
    while number:
        digits.append(digs[number % base])
        number /= base
    if sign < 0:
        digits.append('-')
    digits.reverse()
    return ''.join(digits)


def extract_jwplayer_setup(data):
    """
    Extracts jwplayer setup configuration and returns it as a dictionary.

    :param data: A string to extract the setup from
    :return: A dictionary containing the setup configuration
    """
    data = re.search(r'<script.+?}\(\'(.+)\',\d+,\d+,\'([\w\|]+)\'.*</script>', data, re.I | re.S)
    if data:
        replacements = data.group(2).split('|')
        data = data.group(1)
        for i in reversed(range(len(replacements))):
            if len(replacements[i]) > 0:
                data = re.sub(r'\b%s\b' % int_to_base(i, 36), replacements[i], data)
        data = re.search(r'\.setup\(([^\)]+?)\);', data)
        if data:
            return json.loads(data.group(1).decode('string_escape'))
        #return demjson.decode(data.group(1).decode('string_escape')) ### III
    return None

def datetime_from_utc_to_local(utc_datetime):
    now_timestamp = time.time()
    offset = datetime.datetime.fromtimestamp(now_timestamp) - datetime.datetime.utcfromtimestamp(now_timestamp)
    return utc_datetime + offset

#def parse_html(url):
#    return BeautifulSoup(request(url), 'html5lib', from_encoding='utf-8')

if __name__ == "__main__":
    s = open("The LEGO Ninjago Movie (2017).lv.vtt").read()
    s2 = vtt2srt(s)
    open("The LEGO Ninjago Movie (2017).lv.srt", "w").write(s2)

    s = 'B\xc4\x93thovena D\xc4\x81rgumu Taka (2014)/Beethoven&#x27;s Treasure [LV]'
    #s = s.decode("utf8")
    #s=unescape(s)
    #url = "http://localhost:88/https://walterebert.com/playground/video/hls/ts/480x270.m3u8?token=xxxx~User-Agent=Enigma2~Cookie=xxxxx"
    url = "http://hyt4d6.vkcache.com/secip/0/UMQ3q2gNjTlOPnEVm3iTiA/ODAuMjMyLjI0MC42/1479610800/hls-vod-s3/flv/api/files/videos/2015/09/11/144197748923a22.mp4.m3u8http://hyt4d6.vkcache.com/secip/0/Y-ZA1qRm8toplc0dN_L6_w/ODAuMjMyLjI0MC42/1479654000/hls-vod-s3/flv/api/files/videos/2015/09/11/144197748923a22.mp4.m3u8"
    headers = {"User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 9_2 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/47.0.2526.70 Mobile/13C71 Safari/601.1.46"}
    url = "http://str1e.lattelecom.tv/mobile-vod/mp4:sf_fantastic_beasts_and_where_to_find_them_en_hd.mp4/playlist.m3u8?resource_id=fantastic_beasts_and_where_to_find_them&auth_token=6NAvMFDG+rYTAc4hb5JeL2bmsaRR7bAE23M6KDmhKYOGyXoo0gDpJUE9scYy+nQmfbgk03cWMe9MuXWSH1GqwolEk2jOQ/8Mrg7tOdbwrA8zM7nmkfCZPqQkwajZN4mfSJQVKHqXqJ8="
    headers={}
    print url
    url = "replay::tiesraide/ltv1/"
    url = "ltc::content/live-streams/103?include=quality"
    urlp = streamproxy_encode2(url,headers)
    print urlp
    url2,headers2 = streamproxy_decode2(urlp)
    print url2
    player(urlp)
    pass