content/sources/replay.py

#!/usr/bin/env python
# coding=utf8
#
# This file is part of PlayStream - enigma2 plugin to play video streams from various sources
# Copyright (c) 2016 ivars777 (ivars777@gmail.com)
# Distributed under the GNU GPL v3. For full terms see http://www.gnu.org/licenses/gpl-3.0.en.html
#

try:
    import json
except:
    import simplejson as json
import os
import urllib2, urllib
import datetime, re, sys
import ssl
if "_create_unverified_context" in dir(ssl):
    ssl._create_default_https_context = ssl._create_unverified_context

from SourceBase import SourceBase
import util

API_URL = 'https://replay.lsm.lv/%s/'
headers2dict = lambda  h: dict([l.strip().split(": ") for l in h.strip().splitlines()])
headers0 = headers2dict("""
User-Agent: Mozilla/5.0 (Linux; U; Android 4.4.4; Nexus 5 Build/KTU84P) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30
""")
import HTMLParser
h = HTMLParser.HTMLParser()
from YouTubeVideoUrl import YouTubeVideoUrl

class Source(SourceBase):

    def __init__(self,country="lv",cfg_path=None):
        self.name = "replay"
        self.title = "Replay.lv (LTV)"
        self.img = "replay.png"
        self.desc = "LSM replay.lv satura skatīšanās"

        self.country=country
        self.pic_size = "327x250" #"1000x765"

    def get_content(self, data):
        print "[replay] get_content:", data
        if "::" in data:
            data = data.split("::")[1]
        path = data.split("?")[0]
        clist = path.split("/")[0]
        params = data[data.find("?"):] if "?" in data else ""
        qs = dict(map(lambda x:x.split("="),re.findall("\w+=\w+",params)))
        lang = qs["lang"] if "lang" in qs else self.country

        content=[]
        content.append(("..return", "back","back.png","Return back"))

        if clist=="home":
            content.extend([
                ("Live streams", "replay::tiesraide","","TV live streams"),
                ("Search LV", "replay::search/?term={0}&lang=lv","","Search content LV"),
                ("Last videos LV", "replay::visi/jaunakie/?source=ltv&lang=lv","","Last aired videos LV"),
                ("Last videos by categories LV", "replay::kategorijas/?lang=lv","","Last videos by categories LV"),
                ("All programs LV", "replay::raidijumi/?type=video","","All programs by name LV"),
                ("Programs by categories LV", "replay::categories?lang=lv","","All programs by categories LV"),
                #("Channels", "replay::channels?language=%s"%self.country,"","TV channels"),
                ("Videos by popularity LV", "replay::visi/popularie/?source=ltv&lang=lv","","Programs by popularity"),

                ("Search RU", "replay::search/?term={0}&lang=ru","","Search content RU"),
                ("Last videos RU", "replay::vse/novie/?source=ltv&lang=ru","","Last aired videos RU"),
                ("Last videos by categories RU", "replay::kategorijas/?lang=ru","","Last videos by categories RU"),
                ("All programs RU", "replay::peredachi/?lang=ru&type=video","","All programs by name"),
                ("Programs by categories RU", "replay::categories?lang=ru","","Programs by categories RU")
            ])
            return content

        ### programmu kategorijas ###
        elif clist=="categories":
            url = "https://replay.lsm.lv/lv/raidijumi/?lang=lv&type=video" if lang =="lv" else "https://replay.lsm.lv/ru/peredachi/?lang=ru&type=video"
            r = self._http_request(url)
            icons = {
                #TODO
                "1": "https://www.iconfinder.com/icons/314161/download/png/256",
            }
            for item in re.findall(r'<a .+href="(\?lang=\w+&type=video&theme=\d+)">([^<]+)</a>\t', r):
                title = item[1]
                data2 = url.split("?")[0]+item[0]
                data2 = data2.replace(API_URL%lang,"")
                theme =  item[0].split("theme=")[1]
                img = self.img # TODO icons["1"]
                desc = title
                content.append((title,self.name+"::"+data2,img,desc))
            return content

        ### jaunāko raidijumu kategorijas ###
        elif clist=="kategorijas":
            url = "https://replay.lsm.lv/lv/" if lang =="lv" else "https://replay.lsm.lv/ru/"
            r = self._http_request(url)
            for item in re.findall(r'<a href="/(lv|ru)/kategorija/(\w+)/">.+?<i class="[^"]+"></i>.+?<span>([^<]+)</span>', r, re.DOTALL):
                title = item[2]
                data2 = "kategorija/%s/?lang=%s"%(item[1],item[0])
                img = self.img
                desc = title
                content.append((title,self.name+"::"+data2,img,desc))
            return content

        ### Tiešraides kanānālu saraksts
        elif path=="tiesraide":
            url2 = "https://replay.lsm.lv/styles/main.css"
            r2= self._http_request(url2)
            url = "https://replay.lsm.lv/lv/tiesraide/ltv1/"
            # <div class="visible-xs">
            r= self._http_request(url)
            i2 = r.find('<div class="visible-xs">')
            result = re.findall(r"""onclick="RE.tools.navigate\('/lv/(tiesraide/\w+/)'\).+?<h5>([^<]+)</h5>.+?<time>([^<]+)</time>""", r[0:i2], re.DOTALL)
            for item in result:
                data2 = item[0]
                ch = data2.split("/")[1]
                title = ch.upper()
                m = re.search(r'channel-logo--%s{background-image:url\("([^"]+)"\)' % ch, r2, re.DOTALL)
                img = "https://replay.lsm.lv" + m.group(1)
                desc = item[1]
                # 00:10 &ndash; 00:45
                desc = desc + " (%s)" % item[2].replace("&ndash;", "-")
                title = title + " - " + desc
                content.append((title,self.name+"::"+data2,img,desc))
            return content

        ### Kanāla tiesraide
        elif clist == "tiesraide" and "/" in data:
            ch = data.split('/')[1]
            veids = "audio" if "lr" in ch else "video"
            #url = "https://replay.lsm.lv/lv/tiesraide/ltv7/"
            url = "https://replay.lsm.lv/lv/tiesraide/%s/"%ch
            r= self._http_request(url)

            m = re.search('%s/">.+?<h5>([^<]+)+</h5>.*?<time>([^<]+)</time>'%ch, r, re.DOTALL)
            tagad = m.group(1).strip() if m else ""
            laiks = m.group(2).strip() if m else ""
            laiks = h.unescape(laiks).encode("utf8")
            m = re.search("<h1>([^<]+)</h1>", r)
            title = m.group(1).strip() if m else path.split("/")[1].upper()
            title = "%s - %s (%s)"%(title,tagad,laiks)

            if veids == "video":
                m = re.search('<div class="video"><iframe.+src="([^"]+)"', r)
                if not m:
                    raise Exception("No stream found")
                url = m.group(1)
                headers = headers2dict("""
            User-Agent: Mozilla/5.0 (Linux; U; Android 4.4.4; Nexus 5 Build/KTU84P) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30
            Referer: https://replay.lsm.lv/lv/ieraksts/ltv/70398/tiesa-runa.-lielbritanija-gatavojas-referendumam-par-tu/
                    """)
                r = self._http_request(url,headers=headers)

                #m = re.search('<div class="video-player"><iframe.+src="([^"]+)"', r)
                m = re.search(r'iframe src=\\"([^"]+)\\', r)
                if not m:
                    raise Exception("No stream found")
                url = m.group(1).replace('\\/', "/")
                m = re.search('poster":"([^"]+)"', r, re.DOTALL)
                img =  "https://ltv.lsm.lv" + m.group(1).replace("\\","") if m else ""

                r = self._http_request(url,headers=headers)
                m = re.search('"([^"]+m3u8[^"]+)"', r)
                if not m:
                    raise Exception("No stream found")
                data2 = m.group(1).replace("\\","")
                #r = self._http_request(data2, headers=headers)

            else: # audio
                lrn = ch.replace("lr","")
                url = "https:///www.latvijasradio.lsm.lv/lv/tiesraide/?channel=%s"%lrn
                r = self._http_request(url)
                m = re.search('"file":"([^"]+?m3u8.*?)"', r)
                if not m:
                    raise Exception("No stream found")
                data2 = m.group(1).replace("\\","")
                img = self.img
                desc = ""

            desc = title
            content =(title,data2,img,desc)
            return content

        #m = re.search(r'(\?page=\d+)" class=" paging__prev', r, re.IGNORECASE)
        #if m:
        #    data = re.sub("\?page=\d+", "", data)
        #    data2 = data+m.group(1)
        #    content.append(("Previous page",self.name+"::"+data2,"","Previous page"))

        r = self.call(data, lang=lang)
        if not r:
            return content

        if clist == "search":
            #for r2 in re.findall('<article itemtype="http://schema.org/Article" itemscope class="thumbnail thumbnail--default ">(.+?)</article>', r2, re.DOTALL):
            for item in re.findall('itemprop="image" data-image="([^"]+)".+?<figcaption><h5 itemprop="name"><a itemprop="url" href="([^<]+)">([^<]+)</a></h5></figcaption>', r):
                title = item[2]
                data2 = item[1].replace("/%s/"%lang,"")+"?lang=%s"%lang
                img = "https://replay.lsm.lv" + item[0]
                desc  = title
                content.append((title,self.name+"::"+data2,img,desc))

            #for item in re.findall('itemprop="image" data-image="([^"]+)".+?<figcaption><h4 itemprop="about"><a href="([^"]+)">([^<]+)</a></h4>.*?<h5 itemprop="name"><a itemprop="url" href="([^"]+)">([^<]+)</a></h5>.+?datetime="([^"]+)" class="thumbnail__date ">([^<]+)</time>', r2):
            for item in re.findall('itemprop="image" data-image="([^"]+)".+? class="icon-(ltv|lr).+?<figcaption><h4 itemprop="about"><a href="([^"]+)">([^<]+)</a></h4>.*?<h5 itemprop="name"><a itemprop="url" href="([^"]+)">([^<]+)</a></h5>.+?datetime="([^"]+)" class="thumbnail__date ">([^<]+)</time>', r):
                if item[1]=="lr":continue
                title = "%s - %s (%s)"%(item[3],item[5],item[7])
                data2 = item[4].replace("/%s/"%lang,"")+"?lang=%s"%lang
                img = item[0]  #.replace("https:","http:")
                img = img.replace("lv//", "lv/")  # TODO eksperiments
                desc = title
                content.append((title,self.name+"::"+data2,img,desc))

        ### Raidijumi (programmas) ###
        elif clist in ( "raidijumi","peredachi"):
            for item in re.findall('<li itemprop="name"><a href="([^"]+)" itemprop="url">([^<]+)', r):
            #for item in re.findall('<li itemprop="name"><a href="([^"]+)" itemprop="url">([^<]+)</a></li>', r):
                title = item[1].strip()
                data2 = item[0].replace("/%s/"%lang,"")+"?lang=%s"%lang
                img = ""
                desc  = ""
                content.append((title,self.name+"::"+data2,img,desc))

        ### Raidijuma ieraksti speciālie###
        elif clist in ( "visi","vse",):
            for item in re.findall('(?i)<figure><a href="([^"]+)" itemprop="image" data-image="([^"]+)".+class="thumbnail__duration">([^<]+)</time></figure><figcaption><h4 itemprop="about"><a href="[^"]+">([^<]+)</a></h4>.+>([^<]+).*</h5>.+>([^<]+)</time></figcaption>', r):
                title = item[3].strip()
                data2 = item[0].replace("/%s/"%lang,"")+"?lang=%s"%lang
                img = item[1]  #.replace("https:","http:")
                img = img.replace("lv//", "lv/")  # TODO eksperiments
                desc  = "%s - %s\n%s"%(item[5],item[2],item[4])
                content.append((title,self.name+"::"+data2,img,desc))

        ### Raidijuma ieraksti (videos)
        elif clist in ("raidijums","peredacha","kategorija"):
            for item in re.findall('<article .+ href="([^"]+)".+image="([^"]+)".+class="thumbnail__duration">([^<]+).+">([^<]+).+class="thumbnail__date ">([^"]+)</time></figcaption></article>', r):
                title = item[3].strip()
                data2 = item[0].replace("/%s/"%lang,"")+"?lang=%s"%lang
                img = item[1]  #.replace("https:","http:")
                img = img.replace("lv//", "lv/")  # TODO eksperiments
                desc = "%s - %s"%(item[4],item[2])
                desc = desc.strip()
                content.append((title,self.name+"::"+data2,img,desc))

        ### Ieraksts (video) ###
        elif clist in ("ieraksts","statja"):
            m = re.search('src="([^"]+)"></iframe>', r)
            if m:
                url2 = m.group(1)
                m = re.search('<meta property="og:title" content="([^"]+)"', r, re.DOTALL)
                title = m.group(1)
                m = re.search('<meta property="og:image" content="([^"]+)"', r, re.DOTALL)
                img = m.group(1)
                img = img.replace("lv//", "lv/")  # TODO eksperiments
                m = re.search('<meta property="og:description" content="([^"]+)"', r, re.DOTALL)
                desc = m.group(1)
                m = re.search('<div class="media-description">(.+)<div id="tab-comments">', r, re.DOTALL)
                if m:
                    rr = re.sub("<.+?>", " ", m.group(1))
                    rr = re.sub("[\t\n]", " ", rr)
                    desc =  re.sub(" {2,100}", " ", rr).strip()

                headers = headers2dict("""
User-Agent: Mozilla/5.0 (Linux; U; Android 4.4.4; Nexus 5 Build/KTU84P) AppleWebkit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30
Referer: https://replay.lsm.lv/lv/ieraksts/ltv/70398/tiesa-runa.-lielbritanija-gatavojas-referendumam-par-tu/
            """)
                r2 = self._http_request(url2,headers=headers)
                m = re.search('"file":"([^"]+)', r2)
                if m:
                    data2 = m.group(1).replace("\\","")
                    m = re.search('"idstring":"([^"]+)', r2)
                    #title = m.group(1).strip() if m else ""
                    #title = title.decode("unicode-escape").encode("utf8")
                    #title = title.replace("\n","")
                    #img = ""
                    #desc = ""
                    if "youtube" in data2:
                        video_id = re.search(r"/watch\?v=([^&]+)",data2).group(1)
                        data2 = YouTubeVideoUrl().extract(video_id)
                        if not data2:
                            content=("No stream found %s"%data,"","","No stream found")
                            return content
                    content =(title,data2,img,desc)
                    return content
            content=("No stream found %s"%data,"","","No stream found")
            return content

        m = re.search(r'href="\?([^"]+)" class=" paging__next', r)
        if m:
            page = int(re.search("page=(\d+)",m.group(1)).group(1))
            if "page="in data:
                data2 = re.sub("page=\d+","page=%i"%page,data)
            else:
                if "?" in data:
                    data2 =data+"&page=%i"%page
                else:
                    data2 =data+"?page=%i"%page
            content.append(("Next page",self.name+"::"+data2,"next.png","Next page"))

        return content

    def is_video(self,data):
        if "::" in data:
            data = data.split("::")[1]
        cmd = data.split("/")
        if cmd[0] in ("ieraksts","statja"):
            return True
        elif cmd[0]=="tiesraide" and len(cmd)>1:
            return True
        else:
            return False

    def call(self, data,headers=headers0,lang=""):
        if not lang: lang = self.country
        url = API_URL%lang + data
        #print "[TVPlay Api] url: ",url
        result = []
        content = self._http_request(url,headers=headers0)
        return content


if __name__ == "__main__":
    sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
    import run
    source = Source()
    data= sys.argv[1] if len(sys.argv)>1 else source.name+"::home"
    run.run(source, data)
    sys.exit()