content/sources/replay.py

#!/usr/bin/env python
# coding=utf8
#
# This file is part of PlayStream - enigma2 plugin to play video streams from various sources
# Copyright (c) 2016 ivars777 (ivars777@gmail.com)
# Distributed under the GNU GPL v3. For full terms see http://www.gnu.org/licenses/gpl-3.0.en.html
#

try:
    import json
except:
    import simplejson as json
import os
import urllib2, urllib
import datetime, time, re, sys
import ssl
if "_create_unverified_context" in dir(ssl):
    ssl._create_default_https_context = ssl._create_unverified_context

from SourceBase import SourceBase
import util

API_URL = 'https://replay.lsm.lv/%s/'
headers2dict = lambda  h: dict([l.strip().split(": ") for l in h.strip().splitlines()])
headers0 = headers2dict("""
User-Agent: Mozilla/5.0 (Linux; U; Android 4.4.4; Nexus 5 Build/KTU84P) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30
""")
import HTMLParser
h = HTMLParser.HTMLParser()
from YouTubeVideoUrl import YouTubeVideoUrl

class Source(SourceBase):

    def __init__(self,country="lv",cfg_path=None):
        self.name = "replay"
        self.title = "Replay.lv (LTV)"
        self.img = "replay.png"
        self.desc = "LSM replay.lv satura skatīšanās"

        self.country=country
        self.pic_size = "327x250" #"1000x765"

    def get_content(self, data):
        print "[replay] get_content:", data
        source, data, path, plist, clist, params, qs = self.parse_data(data)
        lang = qs["lang"] if "lang" in qs else self.country
        content=[]
        content.append(("..return", "back","back.png","Return back"))

        if clist=="home":
            content.extend([
                ("Live streams", "replay::tiesraide","","TV live streams"),
                ("Search LV", "replay::meklet?term={0}&lang=lv","","Search content LV"),
                ("Last videos LV", "replay::visi/jaunakie?mtype=video&page=1&lang=lv","","Last aired videos LV"),
                ("Last videos by categories LV", "replay::kategorijas?lang=lv","","Last videos by categories LV"),
                ("Last chance LV", "replay::visi/pedeja-iespeja?lang=lv","","Last videos by categories LV"),
                ("Videos by popularity LV", "replay::visi/popularie?mtype=video&lang=lv","","Programs by popularity"),
                ("All programs LV", "replay::raidijumi?mtype=video&lang=lv","","All programs by name LV"),
                ("Programs by categories LV", "replay::categories?lang=lv","","All programs by categories LV"),
                #("Channels", "replay::channels?language=%s"%self.country,"","TV channels"),

                ("Search RU", "replay::poisk?term={0}&lang=ru","","Search content RU"),
                ("Last videos RU", "replay::vse/novie?mtype=video&lang=ru","","Last aired videos RU"),
                ("Last videos by categories RU", "replay::kategorijas?lang=ru","","Last videos by categories RU"),
                ("All programs RU", "replay::peredachi?lang=ru&mtype=video","","All programs by name"),
                ("Programs by categories RU", "replay::categories?lang=ru","","Programs by categories RU")
            ])
            return content

        ### programmu kategorijas ###
        elif clist=="categories":
            url = "https://replay.lsm.lv/lv/raidijumi" if lang =="lv" else "https://replay.lsm.lv/ru/peredachi"
            r = self._http_request(url)
            icons = {
                #TODO
            "1": "https://www.iconfinder.com/icons/314161/download/png/256",
            }
            result = re.findall(r'<a href="https://replay\.lsm\.lv/(lv|ru)/([^\?^"]+\?category=\d+)" class="badge badge-pill">([^<]+)</a>', r)
            for item in result:

                title = item[2].strip()
                data2 = item[1].replace("&amp;", "&") + "&lang=%s" % item[0]
                #theme =  item[0].split("theme=")[1]
                img = self.img # TODO icons["1"]
                desc = title
                content.append((title,self.name+"::"+data2,img,desc))
            return content

        ### jaunāko raidijumu kategorijas ###
        elif clist=="kategorijas":
            url = "https://replay.lsm.lv/lv/" if lang =="lv" else "https://replay.lsm.lv/ru/"
            r = self._http_request(url)
            for m in re.finditer(r'<a href="https://replay\.lsm\.lv/(lv|ru)/(kategorija/[\w-]+)" class="">.+?<span>([^<]+)</span>', r, re.DOTALL):
                title = m.group(3)
                data2 = "%s?lang=%s"%(m.group(2), m.group(1))
                img = self.img
                desc = title
                content.append((title,self.name+"::"+data2,img,desc))
            return content


        ### Raidijumi (programmas) ###
        elif clist in ( "raidijumi","peredachi"):
            r = self.call(data, lang=lang)
            result = re.findall(r'<a href="https://replay\.lsm\.lv/(lv|ru)/([^"]+)"><span>([^<]+)</span>', r, re.DOTALL)
            for item in result:
                title = item[2].strip()
                data2 = item[1]+"?lang=%s"%item[0]
                img = ""
                desc  = title
                content.append((title,self.name+"::"+data2,img,desc))
            return content

        ### Tiešraides kanānālu saraksts
        elif path=="tiesraide":
            url = "https://replay.lsm.lv/lv/tiesraide/ltv1/"
            # <div class="visible-xs">
            r = self._http_request(url)
            result = re.findall(r'class="small-guide.+?href="https://replay\.lsm\.lv/lv/([^"]+)".+?title="([^"]+)".+?<div class="item-body">([^<]+)</div>.+?<time>([^<]+)</time>', r, re.DOTALL)
            for item in result:
                data2 = item[0]
                ch = data2.split("/")[1]
                title = item[1]
                img = ch + ".png"
                desc = item[2].strip()
                desc = desc + " [%s]" % item[3]
                title = title + " - " + desc
                content.append((title,self.name+"::"+data2,img,desc))

            url = "https://ltv.lsm.lv/lv/tieshraide/"
            r = self._http_request(url)
            result = re.findall('<article class="entry highlight"><figure><a href="/lv/([^"]+)"><img src="([^"]+)".+?>([^<]+)</a>', r, re.IGNORECASE | re.DOTALL | re.MULTILINE)
            for item in result:
                data2 = item[0]
                ch = data2.split("/")[1]
                title = item[2] + " (ltv.lsm.lv)"
                img = "https://ltv.lsm.lv/" + item[1]
                desc = title
                content.append((title,self.name+"::"+data2,img,desc))
            return content

        ### Meklēsana ###
        # TODO (jāsakrārto pēc jaunā templeita)
        elif clist in ("search"):
            r = self.call(data, lang=lang)
            if not r:
                return content
            #for r2 in re.findall('<article itemtype="http://schema.org/Article" itemscope class="thumbnail thumbnail--default ">(.+?)</article>', r2, re.DOTALL):
            for item in re.findall('itemprop="image" data-image="([^"]+)".+?<figcaption><h5 itemprop="name"><a itemprop="url" href="([^<]+)">([^<]+)</a></h5></figcaption>', r):
                title = item[2]
                data2 = item[1].replace("/%s/"%lang,"")+"?lang=%s"%lang
                img = "https://replay.lsm.lv" + item[0]
                desc  = title
                content.append((title,self.name+"::"+data2,img,desc))

            #for item in re.findall('itemprop="image" data-image="([^"]+)".+?<figcaption><h4 itemprop="about"><a href="([^"]+)">([^<]+)</a></h4>.*?<h5 itemprop="name"><a itemprop="url" href="([^"]+)">([^<]+)</a></h5>.+?datetime="([^"]+)" class="thumbnail__date ">([^<]+)</time>', r2):
            for item in re.findall('itemprop="image" data-image="([^"]+)".+? class="icon-(ltv|lr).+?<figcaption><h4 itemprop="about"><a href="([^"]+)">([^<]+)</a></h4>.*?<h5 itemprop="name"><a itemprop="url" href="([^"]+)">([^<]+)</a></h5>.+?datetime="([^"]+)" class="thumbnail__date ">([^<]+)</time>', r):
                if item[1]=="lr":continue
                title = "%s - %s (%s)"%(item[3],item[5],item[7])
                data2 = item[4].replace("/%s/"%lang,"")+"?lang=%s"%lang
                img = item[0]  #.replace("https:","http:")
                img = img.replace("lv//", "lv/")  # TODO eksperiments
                desc = title
                content.append((title,self.name+"::"+data2,img,desc))

        # Raidijumu saraksti
        elif True:
            r = self.call(data, lang=lang)
            if not r:
                return content
            result = re.findall("<article.+?>(.+?)</article>", r, re.DOTALL)
            for item in result:
                m = re.search(r'<h4 class="thumbnail__title"><a href="https://replay\.lsm\.lv/(\w+)/([^"]+)">([^<]+)</a></h4>', item, re.DOTALL)
                if not m:
                    continue
                title = m.group(3).strip()
                desc = title
                data2 = m.group(2)
                lang = m.group(1)
                m = re.search(r'style="background-image: url\(([^\)]+)\)"', item, re.DOTALL)
                img = m.group(1) if m else ""
                m = re.search('datePublished" datetime="([^"]+)"', item, re.DOTALL)
                if m:
                    published = m.group(1)
                    try:
                        published = util.datetime_from_zulu_to_utc(published)
                        published = util.datetime_from_utc_to_local(published)
                        published_datetime = datetime.datetime.strftime(published,"%d.%m.%Y %H:%M")
                        published_date = datetime.datetime.strftime(published,"%d.%m.%Y")
                        title = title + " (%s)" % published_date
                        desc = desc + "\nPublished: %s" % published_datetime
                    except:
                        a = 1
                        pass
                m = re.search('thumbnail__duration">([^<]+)<', item, re.DOTALL)
                if m:
                    desc = desc + "\nDuration: %s" % m.group(1)
                m = re.search(r'<a href="https://replay\.lsm\.lv/lv/raidijums/[^"]+">([^<]+)</a>', item, re.DOTALL)
                if m:
                    desc = desc + "\nProgram: %s" % m.group(1)
                m = re.search('<li class=".*?">(Vēl[^<]+)</li>', item, re.DOTALL)
                if m:
                    desc = desc + "\nRemaining: %s" % m.group(1)
                content.append((title,self.name+"::"+data2,img,desc))

            m = re.search(r'page=(\d+)" rel="next"', r, re.DOTALL)
            if m:
                page = int(m.group(1))
                if "page="in data:
                    data2 = re.sub("page=\d+","page=%s"%page,data)
                else:
                    if "?" in data:
                        data2 =data+"&page=%s"%page
                    else:
                        data2 =data+"?page=%s"%page
                content.append(("Next page",self.name+"::"+data2,"next.png","Next page"))

            return content

    #===========================================================================
    def get_streams(self, data):
        source, data, path, plist, clist, params, qs = self.parse_data(data)
        lang = "ru" if clist == 'statja' else "lv"
        headers2 = {}
        nfo = {}

        ### Kanāla tiesraide
        if clist == "tiesraide" and "/" in data or clist == "tieshraide":
            if clist == "tiesraide":
                url = "https://replay.lsm.lv/lv/" + data
                #r = self.call(data, lang=lang)
            else:
                url = "https://ltv.lsm.lv/lv/" + data
            r = self._http_request(url)
            ch = data.split('/')[1]
            veids = "audio" if "lr" in ch else "video"
            img = ch + ".png"

            m = re.search('span class="badge badge-primary">Tagad</span>([^<]+)<.+?<time>([^<]+)</time>', r, re.DOTALL)
            if clist == "tieshraide":
                m = re.search("<h1>([^<]+)</h1>", r)
                title = m.group(1).strip() if m else ch.upper()
                desc = title
            else:
                tagad = m.group(1).strip() if m else ""
                laiks = m.group(2).strip() if m else ""
                laiks = h.unescape(laiks).encode("utf8")
                title = m.group(1).strip() if m else path.split("/")[1].upper()
                dd = datetime.datetime.now().strftime("%d.%m.%Y")
                title = "%s - %s (%s) [%s]"%(ch.upper(), title, dd, laiks)
                desc = title

            if veids == "video":
                m = re.search('<iframe.+?src="([^"]+)"', r)
                if not m:
                    raise Exception("No stream found")
                headers = headers2dict("""
            User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0
            Referer: %s
                    """% url)
                url = m.group(1)
                r = self._http_request(url,headers=headers)

                #m = re.search('<div class="video-player"><iframe.+src="([^"]+)"', r)
                m = re.search(r'iframe src=\\"([^"]+)\\', r)
                if m:
                    #raise Exception("No stream found")
                    headers["Referer"] = url
                    url2 = m.group(1).replace('\\/', "/")
                    m = re.search('poster":"([^"]+)"', r, re.DOTALL)
                    img =  "https://ltv.lsm.lv" + m.group(1).replace("\\","") if m else ch + '.png'
                    r = self._http_request(url2,headers=headers)
                    url = url2
                m = re.search('"([^"]+m3u8[^"]+)"', r)
                if not m:
                    raise Exception("No stream found")
                data2 = m.group(1).replace("\\","")
                headers2["Referer"] = url
                headers2["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36"
                headers2["Origin"] = "https://embed.lsm.lv"
                headers2["Host"] = data2.split("/")[2]
                #r = self._http_request(data2, headers=headers2)
                a = 1

            else: # audio
                lrn = ch.replace("lr","")
                url = "https://latvijasradio.lsm.lv/lv/tiesraide/?channel=%s"%lrn
                r = self._http_request(url)
                m = re.search('"file":"([^"]+?m3u8.*?)"', r)
                if not m:
                    raise Exception("No stream found")
                data2 = m.group(1).replace("\\","")
                img = ch + ".png"


        ### Ieraksts (video) ###
        elif clist in ("ieraksts","statja"):
            r = self.call(data, lang=lang)
            m = re.search('<iframe src="([^"]+)"', r)
            if not m:
                raise Exception("No stream found")
            url2 = m.group(1).replace("&amp;", "&")
            #m = re.search('<h1 class="title">([^<]+)</h1>', r)
            m = re.search('<meta name="name" content="([^"]+)">', r)
            title = m.group(1)
            title = title.replace("- REplay.lv", "")
            title = title.strip()
            m = re.search('<meta content="([^"]+)" property="og:video:release_date"', r)
            if m:
                dd = m.group(1).split("T")[0].split("-")
                title += " (%s.%s.%s)" % (dd[2], dd[1], dd[0])
            m = re.search('<meta content="([^"]+)" property="og:image">', r)
            img = m.group(1) if m else ""
            m = re.search('<div class="description">(.+?)</div>', r, re.DOTALL)
            if m:
                plot = m.group(1)
                rr = re.sub("<.+?>", " ", m.group(1))
                rr = re.sub("[\t\n]", " ", rr)
                plot =  re.sub(" {2,100}", " ", rr).strip()
            else:
                plot = ""
            desc = [title]
            m = re.search(r'<figure><span class="channel-logo (\w+)"></span></figure>([^<]+)</a>', r, re.IGNORECASE)
            if m:
                desc.append("Programma: %s [%s]" % (m.group(2), m.group(1).upper()))
            m = re.search('<ul class="media-info-list">(.+?)</ul>', r, re.DOTALL)
            if m:
                for m2 in re.finditer("<strong>([^<]+)</strong> *(<a[^<]+?>)*([^<]+)*<", m.group(1)):
                    desc.append("%s %s" % (m2.group(1).strip(),m2.group(3).strip() if m2.group(3) else ""))
                m2 = re.search('li class="[^"]*">(Pieejam.+)</li>', m.group(1))
                if m2:
                    desc.append(m2.group(1))
            desc.append(plot)
            desc = "\n".join(desc)

            nfo["title"] = title
            nfo["thumb"] = img
            #nfo["runtime"] = None
            #nfo["quality"] = None
            nfo["plot"] = desc
            nfo["tagline"] = plot


            headers = headers2dict("""
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36
Referer: https://replay.lsm.lv/%s/%s
            """% (lang, data))
            r2 = self._http_request(url2,headers=headers)
            m = re.search(r"<script>LTV.Video.players.create\('videoplay', ({.+?})\);</script>", r2, re.DOTALL)
            #m = re.search('"file":"([^"]+)', r2)
            if m:
                js = json.loads(m.group(1))
                if js["ext_video_id"]:
                    data2 = "https://www.youtube.com/watch?v=" + js["ext_video_id"]
                elif "playlist" in js["player"]:
                    data2 = js["player"]["playlist"][0]["sources"][0]["file"]
                elif "clip" in js["player"]:
                    if len(js["player"]["clip"]["sources"]) > 0:
                        data2 = js["player"]["clip"]["sources"][0]["src"]
                    else:
                        raise Exception("No streams found!")
                elif "file" in  js["player"]:
                    data2 = js["player"]["file"]
                else:
                    raise Exception("No stram found")
            else:  # futbols u.c.
                m = re.search('<div class="video-player">.+?<iframe src="([^"]+)', r2, re.DOTALL)
                if not m:
                    raise Exception("No stream found")
                url3 = m.group(1)
                if url3.startswith("//"):
                    url3 = "https:" + url3
                headers["Referer"] = url2
                r3 = self._http_request(url3, headers=headers)
                if not r3:
                    raise Exception("No stream found")
                m = re.search(r"src: '([^']+)',\s+?type: 'application/x-mpegurl'", r3, re.DOTALL|re.IGNORECASE)
                if not m:
                    raise Exception("No stream found")
                data2 = "https:" + m.group(1)


            if "youtube" in data2:
                video_id = re.search(r"/watch\?v=([^&]+)",data2).group(1)
                data2 = YouTubeVideoUrl().extract(video_id)
                if not data2:
                    raise Exception("No stream found %s"%data)

        r2 = self._http_request(data2, headers=headers)
        if not r2:
            raise Exception("Stream is not accesible %s"%data)
        streams = []
        slist = re.findall("#EXT-X-STREAM-INF:([^\n]+)\n([^\n]+)", r2, re.DOTALL)
        if not slist:
            slist = [('BANDWIDTH=0,NAME="SD",RESOLUTION=?', data2)]
        for cc in slist:
            m = re.search("RESOLUTION=([\dx]+)",cc[0])
            resolution = m.group(1) if m else "?"
            m = re.search("BANDWIDTH=([\d]+)",cc[0])
            bitrate = int(m.group(1)) if m else 0
            url2 = cc[1].encode("utf8")
            if not url2.startswith("http"):
                uu = data2.split("/")[:-1]
                uu.append(url2)
                url2 = "/".join(uu)

            stream = util.item()
            stream["name"] = title
            stream["url"] = url2
            stream["img"] = img
            stream["desc"] = desc
            stream["resolver"] = "replay"
            stream["headers"] = headers
            stream["nfo"] = {"movie":nfo}
            stream["quality"] = resolution
            stream["order"] = bitrate
            #stream["filename"] = filename # TODO jāģenere jēdzigs faila nosaukums, kuru pēc tam izmanto downloads
            streams.append(stream)
        streams = sorted(streams,key=lambda item: item["order"],reverse=True)
        return streams

    def is_video(self,data):
        if "::" in data:
            data = data.split("::")[1]
        cmd = data.split("/")
        if cmd[0] in ("ieraksts","statja"):
            return True
        elif cmd[0]=="tiesraide" and len(cmd)>1:
            return True
        elif cmd[0] == "tieshraide":
            return True
        else:
            return False

    def call(self, data,headers=headers0,lang=""):
        if not lang: lang = self.country
        url = API_URL%lang + data
        #print "[TVPlay Api] url: ",url
        result = []
        content = self._http_request(url,headers=headers0)
        return content


if __name__ == "__main__":
    sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
    import run
    source = Source()
    data= sys.argv[1] if len(sys.argv)>1 else source.name+"::home"
    run.run(source, data)
    sys.exit()