GetPicons/get_picons.py

# coding: utf-8
#
# This file is part of GetPicons - enigma2 plugin to download picons from lyngsat.com
# Copyright (c) 2016 ivars777 (ivars777@gmail.com)
# Distributed under the GNU GPL v3. For full terms see http://www.gnu.org/licenses/gpl-3.0.en.html

"""
Dowload and create Enigma2 channels' picons files form lyngsat.com

Usage [options]
Options:
    -p PACKAGE_LIST, --package=PACKAGE_LIST     list of package names (html file name in lyngsat),
                                                e.g "viasat,ntvplus36"
    -s  SAT_LIST, --sat=SAT_LIST                list of sattelite positions, e.g. "4.9,-0.8"
    -z  SIZE                                    icon size, e.g. "100x60", default - "220x132"
    -f PATH, --folder PATH                      picon files output folder, default - "/media/hdd/picon"
    -e PATH, --enigma PATH                      enigma2 folder whera lamedb,settings are located, default - "/etc/enigma2"
                                                you can use urls, e.g. "ftp://root@receiver_address/etc/enigma2"
    -o, -overwrite                              overwrite existing picons file (default - no)
    -d, -debug                                  display work progress and write debug info to file for not found services
    -h, --help                                  this help file

(c)Ivars777 (ivars777@gmail.com) 2013-2015, v0.2
"""

import sys, os, os.path, getopt
import re
import urllib2
import requests

try:
    import Image
except:
    from PIL import Image
from StringIO import StringIO
import logging

options = args = None
_sd= lambda x,d: d if x is None else x # set default value
_sl= lambda x: False if x is None else True # set True of False for option

def parse_arguments(argv,opt_short,opt_long):
    "Parse command line arguments"
    try:
        options, args = getopt.gnu_getopt(argv, opt_short,opt_long)
        options = Rec(dict([(o[0].replace("-",""),o[1]) for o in options]))
    except getopt.GetoptError, err:
        print err.msg
        print __doc__
        sys.exit(2)
    if options.has_key("h") or options.has_key("help"):
        print __doc__
        sys.exit(2)
    return options,args

services = None

def main(argv):
    global options, args, services

    # Parsing options
    opt_short = 'p:s:z:f:e:ordh'
    opt_long = ["package=","sat","folder=","enigma=","overwrite","hires","debug","help"]
    options,args = parse_arguments(argv[1:], opt_short, opt_long)
    options.package = _sd(options.package,_sd(options.p,""))
    options.sat = _sd(options.sat,_sd(options.s,""))
    options.enigma = _sd(options.enigma,_sd(options.e,"/etc/enigma2"))
    options.folder = _sd(options.folder,_sd(options.f,"/media/hdd/picon"))
    options.size = _sd(options.size,_sd(options.z,"220x132"))
    options.overwrite = _sl(_sd(options.overwrite,options.o))
    options.debug = _sl(_sd(options.debug,options.d))
    options.hires = _sl(_sd(options.hires,options.r))

    options.w,options.h = map(int,options.size.split("x"))
    if not os.path.exists(options.folder):
        os.makedirs(options.folder)

    if options.debug:
        #filename="get_picons.log" if not os.name == "posix" else "/var/log/get_picons.log"
        filename = os.path.join(options.folder,"get_picons.log")
        FORMAT = "%(asctime)-15s %(message)s"
        logging.basicConfig(filename=filename, format=FORMAT,level=logging.ERROR)

    print "** Analyzing lamedb"
    services = DBServices(options.enigma)
    if options.sat:
        package_list = []
        for sat in options.sat.split(","):
            package_list.extend(get_packages(sat))
    elif options.package:
        package_list = options.package.split(",")
    else: # get sattelite list from Enigma settings
        sat_list = []
        package_list = []
        if not "ftp:" in options.enigma:
            fname = os.path.join(options.enigma,"settings")
            f = open(fname)
        else:
            sep = "" if options.enigma[-1]=="//" else "//"
            fname = options.enigma + sep + "settings"
            f = urllib2.urlopen(fname)
        for line in f:
            if not "config.Nims.0.advanced.sat." in line: continue
            sat_list.append(line.split(".")[5])
        for sat in sat_list:
            package_list.extend(get_packages(sat))

    for package in package_list:
        create_picons(package)

def create_picons(package):
    "Create picons files for package"
    global services,options
    url_package = "https://www.lyngsat.com/packages/%s.html"%package
    if options.debug:
        print "** Downloading and looking %s for picons"%url_package
    num_picons = 0
    num_skipped = 0
    num = 0
    try:
        #soup = html = urllib2.urlopen(url_package).read().decode("latin1")
        soup = html = get_page(url_package).decode("latin1")
    except:
        raise Exception("Package page '%s' does not found"%url_package)
    title = re.search("<title>(.+?)</title>", soup, re.DOTALL | re.IGNORECASE).group(1)
    pos = re.search("\d+\.\d+",title).group().replace(".","")
    ew = re.search("\xb0.",title).group()[1]
    ns = int(pos)
    if ew == "W":
        ns = 3600 - ns
    if ns==48: ns=49
    if ns==359: ns=360
    #pos = "%04x0000"%ns
    #pos_num = int(pos[0:4],16)
    title2 = title[0:title.index("- LyngSat")]
    print "** %s: "%title2.replace(u'\xb0',""),
    tables = re.findall(r"<table width=720 border cellspacing=0 cellpadding=0>\s*(?:(?=<tr>))(.+?)</table>", soup, re.DOTALL | re.IGNORECASE)
    for table in tables:
        tr= html_findall("tr",table)
        for i in range(2,len(tr)-1):
            td= html_findall("td",tr[i])
            if len(td) == 10:
                freq,polar = re.search(r"<b>(\d+) ([HVLR])", tr[i], re.DOTALL | re.IGNORECASE).groups()
                freq = int(freq)
                p_value = {"H":0,"V":1,"L":2,"R":3}
                polar = p_value[polar]
                b=1
            else:
                b=0
            icon_url = html_attr("src",td[b])
            if icon_url:
                #icon_url = "http://www.lyngsat.com" + icon_url.group(1).replace("/icon","/logo").replace(".gif",".jpg")
                # https://www.lyngsat.com/logo/tv/vv/viasat_history.png
                # https://www.lyngsat-logo.com/hires/vv/viasat_history.png
                if options.hires:
                    icon_url = "https://www.lyngsat-logo.com" + icon_url.group(1).replace("/logo/tv", "/hires")
                else:
                    icon_url = "https://www.lyngsat.com" + icon_url.group(1) # simple picture
            else:
                icon_url = ""
            name = html_text(td[b+1]).group(1)
            sid = html_text(td[b+5])
            sid = sid.group(1).replace("&nbsp;","") if sid else ""
            sid = int(sid) if sid and not sid == u"-" else ""
            vpid = html_text(td[b+6])
            vpid = vpid.group(1).replace("&nbsp;","") if vpid else ""
            #try:
            #    vpid = int(vpid)
            #except Exception:
            #    vpid = None
            #vpid = int(vpid) if vpid and not vpid == u"-" else None
            if not sid  or not vpid or icon_url=="":
                continue

            if options.debug:
                print " package:%i/%s: service:%s "%(ns,package, name.encode("ascii","replace")),
            num += 1

            sref = find_sref(ns,freq,polar,sid)
            if not sref:
                sref = find_sref(ns,freq+1,polar,sid)
                if not sref:
                    sref = find_sref(ns,freq-1,polar,sid)
                    if not sref:
                        sref = find_sref(ns, freq, polar-2, sid)
            if not sref:
                if options.debug:
                    print " -- NOK (no sref in lamedb!)"
                    logging.error(u" no sref - package:%i/%s: service:%s pos:%s/%s freq:%i polar:%s sid:%i/%x"%(ns,package, name,ns, pos_int2str(ns),freq,polar,sid,sid))
                continue

            fname = sref.replace(":","_")
            fname  = fname[:-1]+".png"
            fname = os.path.join(options.folder, fname)
            if os.path.exists(fname) and not options.overwrite:
                if options.debug: print " -- skipped"
                num_skipped += 1
                continue

            #try:
            #im = Image.open(StringIO(urllib2.urlopen(icon_url).read()))
            data = get_page(icon_url)
            if not data:
                if options.debug: print " -- NOK (no picon image)"
                continue
            try:
                im = Image.open(StringIO(data))
                im.thumbnail((options.w,options.h), Image.ANTIALIAS)
                im = im.convert('P', palette=Image.ADAPTIVE)
            except Exception:
                im = None
            if not im:
                if options.debug: print " -- NOK (no picon image)"
                continue

            if options.debug: print " -- downloaded"
            im.save(fname,"png")
            del im
            num_picons += 1

    print "%i picons created, %i skipped"%(num_picons,num_skipped)

def get_packages(sat):
    "Get packages list (names) for sattelite from lyngsat.com page"
    global options
    sat= sat.replace(".","")
    num = int(sat)
    if num>1800: num = num - 3600
    if num>0 and num<730: soup = get_soup("europe")
    elif num>730 and num<1600: soup = get_soup("asia")
    elif num<0 and num>-610: soup = get_soup("atlantic")
    elif num<-610 and num>-1600: soup = get_soup("america")
    else:
        return []
    i1,i2 = gr=re.search(r"<table cellspacing=0 border>\s*(?:(?=<tr>))(.+?)</table>", soup, re.DOTALL | re.IGNORECASE).span()
    packages = []
    for s in re.findall("<tr>(.+?)</tr>", soup[i1:i2], re.DOTALL | re.IGNORECASE):
        gr = re.search(r"(\d+.)</font><font size=1>(\d).+#176;([EW])", s,re.DOTALL | re.IGNORECASE).groups()
        pos = int("".join(gr[:-1]).replace(".",""))
        if gr[2] == "W": pos = -pos
        if abs(num - pos) <=2:
            package = re.search(r"www\.lyngsat\.com/packages/(\w.+?)\.html", s, re.DOTALL | re.IGNORECASE).group(1)
            packages.append(package)
    return packages

soups = {}
def get_soup(region):
    if soups.has_key(region):
        return soups[region]
    url = "http://www.lyngsat.com/packages/%s.html"%region
    if options.debug:
        print "** Downloading and looking %s for packages"%url
    try:
        #html = urllib2.urlopen(url).read().decode("latin1")
        html = get_page(url).decode("latin1")
    except:
        raise Exception("Packege page '%s' does not found"%url)
    soups[region] = html
    return soups[region]


def find_sref(ns,freq,polar,sid):
    "Find service reference according to given parameters"
    global services
    tp = services.get_transp_by_pic((ns,freq,polar))
    if tp:
        serv = services.get_service_by_chid((sid,tp["tsid"],tp["onid"]))
        if serv:
            return serv["sref"]

class DBServices(object):
    """Dreambox services, stored in lamedb (E2)/services(E1) file handling class"""

    def __init__(self,service_path,enigma=2):
        self.enigma = enigma
        self.service_path = service_path

        if self.enigma == 2:
            if not "ftp:" in service_path:
                self.service_fname= os.path.join(service_path,"lamedb")
                self.bouquets_fname = os.path.join(service_path, "bouquets.tv")
            else:
                sep = "" if service_path[-1]=="//" else "//"
                self.service_fname= service_path+sep+"lamedb"
                self.bouquets_fname = service_path+sep+"bouquets.tv"

        else: # ENIGMA === "1":
            self.service_fname = os.path.join(service_path,"services")
            self.bouquets_fname = os.path.join(service_path, "userbouquets.tv.epl")

        self.services = []
        self.index_sref= {}
        self.index_chid = {}
        self.index_name = {}
        self.index_provider = {}
        self.bouquets = []
        self.bouquets_services = {}

        self.transp = []
        self.tp_index_tpid = {}
        self.tp_index_pic ={}

        self._load_services()
        #self._load_bouquets()

    def _load_services(self):
        #tpref - lamedb/transponders
        # NS:TSID:ONID
        #     s FREQ:SR:POLAR:FREC:49:2:0
        #  X    X   X
        #         D    D  D     D

        # lref - lamedb/services
        #	SID:NS:TSID:ONID:STYPE:UNUSED(channelnumber in enigma1)
        #	0   1  2    3    4     5
        #	X   X  X    X    D     D
        # wref bouquets/picon
        #   REFTYPE:FLAGS:STYPE:SID:TSID:ONID:NS:PARENT_SID:PARENT_TSID:UNUSED
        #   0       1     2     3   4    5    6  7          8           9
        #   D       D     X     X   X    X    X  X          X           X

        f_services = open(self.service_fname,"r") if not self.service_fname[0:4] == "ftp:" else urllib2.urlopen(self.service_fname)

        line = f_services.readline()
        if not "eDVB services" in line:
            raise Exception("no correct lamedb file")
        line = f_services.readline()

        # Read transponders
        i = 0
        while True:
            line = f_services.readline()
            if  "end" in line: break
            #tp = record.Rec()
            tp = {}
            ff = line.strip().split(":")
            tp["ns"] = ns = pos_str2int(ff[0])
            tp["tsid"] = tsid = int(ff[1],16)
            tp["onid"] = onid = int(ff[2],16)
            line = f_services.readline()
            tp["params"] = params = line.strip().split(" ")[1]
            ff = params.split(":")
            tp["freq"] = freq = int(ff[0][:-3])
            tp["polar"] = polar = int(ff[2])
            tp["tpid"] = tpid = (ns,tsid,onid)
            self.transp.append(tp)
            self.tp_index_tpid[tpid] = i
            self.tp_index_pic[(ns,freq,polar)] = i

            line = f_services.readline()
            i += 1

        # Reading services
        i= 0
        line = f_services.readline()
        while True:

            line = f_services.readline()
            if line[0:3] == "end": break
            if not line: break
            #rec = record.Rec()
            rec = {}
            rec["lref"] = line.lower().strip().decode("utf-8")

            line = f_services.readline()
            #line = line.replace('\xc2\x87', '').replace('\xc2\x86', '')
            #line = decode_charset(line)
            line = line.decode("utf-8")
            rec["name"] = line.strip()

            line = f_services.readline()
            pl = line.split(",")
            provider = ""
            for p in pl:
                if p[0:2] == "p:":
                    provider = p[2:].strip()
                    break
            if provider == "":
                provider = "unknown"
            rec["provider"] = provider.decode("utf-8")

            rec["sref"] = lref2sref(rec["lref"])
            r = lref_parse(rec["lref"])
            rec["stype"] = r["stype"]
            rec["chid"]  = (r["sid"],r["tsid"],r["onid"])
            self.services.append(rec)

            self.index_sref[rec["sref"]] = i
            self.index_chid[rec["chid"]] = i
            name = rec["name"].lower()
            if not self.index_name.has_key(name): self.index_name[name] = []
            self.index_name[name].append(i)
            provider = rec["provider"].lower()
            if not self.index_provider.has_key(provider): self.index_provider[provider] = []
            self.index_provider[provider].append(i)

            i += 1

        f_services.close()

    def _load_bouquets(self):
        f_bouquets = open(self.bouquets_fname,"r") if not self.bouquets_fname[0:4] == "ftp:" else urllib2.urlopen(self.bouquets_fname)
        for line in f_bouquets.readlines():
            if line[0:9] == "#SERVICE:":
                if self.enigma==1:
                    bn = line.strip().split("/")
                else:
                    bn = line.strip().split(":")
                bfn = bn[-1]
                bf = open(os.path.join(self.service_path,bfn))
                for line2 in bf.readlines():
                    if line2[0:5] == "#NAME":
                        bname = line2.strip()[6:]
                        self.bouquets.append(bname)
                        self.bouquets_services[bname] = []
                    elif line2[0:8] == "#SERVICE":
                        sref = line2.strip()[9:]
                        r = sref_parse(sref)
                        if r["flags"] == 64: continue
                        self.bouquets_services[bname].append(sref)
                bf.close()
        f_bouquets.close()

    def get_bouquets(self):
        if not self.bouquets:
            self._load_bouquets()
        return self.bouquets

    def get_bouquet_services(self,bn):
        if not self.bouquets:
            self._load_bouquets()
        return [self.get_service_by_sref(sref) for sref in self.bouquets_services[bn]]

    def get_service_by_sref(self,sref):
        return self.services[self.index_sref[sref]] if self.index_sref.has_key(sref) else None

    def get_service_by_chid(self,chid):
        return self.services[self.index_chid[chid]] if self.index_chid.has_key(chid) else None

    def get_service_by_name(self,name):
        return [self.services[i] for i in self.index_name[name]]

    def get_service_by_provider(self,provider):
        return [self.services[i] for i in self.index_provider[provider]]

    def get_transp_by_tpid(self,tpid):
        return self.transp[self.tp_index_tpid[tpid]] if self.tp_index_tpid.has_key(tpid) else None

    def get_transp_by_pic(self,picid):
        return self.transp[self.tp_index_pic[picid]] if self.tp_index_pic.has_key(picid) else None

def decode_charset(s):
    u = None
    if isinstance(s,unicode):
        return s
    charset_list=('iso-8859-4','iso-8859-1','iso-8859-2''iso-8859-15')

    for charset in charset_list:
        try:
            u=unicode(s,charset,"strict")
        except:
            pass
        else:
            break

    if u == None:
        raise Error, "CHARSET ERROR while decoding lamedb. Aborting !"
    else:
        return(u)

# lref - lamedb/services
#	SID:NS:TSID:ONID:STYPE:UNUSED(channelnumber in enigma1)
#	0   1  2    3    4     5
#	X   X  X    X    D     D
#
# wref bouquets/picon
#   REFTYPE:FLAGS:STYPE:SID:TSID:ONID:NS:PARENT_SID:PARENT_TSID:UNUSED
#   0       1     2     3   4    5    6  7          8           9
#   D       D     X     X   X    X    X  X          X           X

def lref2sref(s):
    "Converts service refefence from lamedb format to bouquets/picon (standard) format"
    f = s.split(":")
    sid = int(f[0],16)
    ns = int(f[1], 16)
    tsid = int(f[2],16)
    onid = int(f[3],16)
    stype = int(f[4])

    s2 = "1:0:%X:%X:%X:%X:%X:0:0:0:" % (stype,sid,tsid,onid,ns)
    return s2

def lref_parse(s):
    "Parse lamedb/service string, return dictionary of items"
    f = s.split(":")
    r = {}
    r["sid"] = int(f[0],16)
    r["ns"] = int(f[1], 16)
    r["tsid"] = int(f[2],16)
    r["onid"] = int(f[3],16)
    r["stype"] = int(f[4])
    return r

def sref2lref(s):
    "Converts service refefence from bouquets/picon (standard) format to lamedb format"
    f = s.split(":")
    reftype = int(f[0])
    flags = int(f[1])
    stype = int(f[2],16)
    sid = int(f[3],16)
    tsid = int(f[4],16)
    onid = int(f[5],16)
    ns = int(f[6],16)

    s2 = "%04x:%08x:%04x:%04x:%i:0" % (sid,ns,tsid,onid,stype)
    return s2

def sref_parse(s):
    "Parse service refefence string , return dictionary of items"
    f = s.split(":")
    r = {}
    r["reftype"] = int(f[0])
    r["flags"] = int(f[1])
    r["stype"] = int(f[2],16)
    r["sid"] = int(f[3],16)
    r["tsid"] = int(f[4],16)
    r["onid"] = int(f[5],16)
    r["ns"] = int(f[6],16)
    return r

pos_int2str = lambda pos_num: "%04x0000"%pos_num
pos_str2int = lambda pos: int(pos[0:4],16)


def decode_charset(s):
    u = None
    charset_list = ('utf-8','iso-8859-1','iso-8859-2','iso-8859-15')

    for charset in charset_list:
        try:
            u = unicode(s,charset,"strict")
        except:
            pass
        else:
            break

    if u == None:
        print("CHARSET ERROR while decoding string")
        sys.exit(1)
    else:
        return(u)
    
headers2dict = lambda  h: dict([l.strip().split(": ") for l in h.strip().splitlines()])
headers = headers2dict("""
User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0
Referer: http://www.lyngsat.com/
""")

def get_page(url):
   r=requests.get(url,headers=headers)
   if r.status_code in (200,304):
        return r.content
   else:
       return ""
    
    
html_text = lambda html: re.search(r">([^<^\n]+?)<", html, re.DOTALL | re.IGNORECASE)
html_tag = lambda tag, html: re.search(r"<%s\b[^>]*>(.*?)</%s>"%(tag,tag), html, re.DOTALL | re.IGNORECASE)
html_attr = lambda attr,html: re.search("""%s="?([^'" >]+)"""%(attr), html, re.DOTALL | re.IGNORECASE)
html_findall_content = lambda tag,html: re.findall(r"<%s\b[^>]*>(.*?)</%s>"%(tag,tag), html, re.DOTALL | re.IGNORECASE)
html_findall = lambda tag,html: re.findall(r"<%s\b[^>]*>.*?</%s>"%(tag,tag), html, re.DOTALL | re.IGNORECASE)

import UserDict
class Rec(UserDict.DictMixin):
    "Dict like class allowing a.x instead of a['x']"
    def __init__(self, dict=None, **kwargs):
        self.__dict__ = {}
        if dict is not None:
            self.update(dict)
        if len(kwargs):
            self.update(kwargs)
    def __setitem__(self, name, value): setattr(self, name, value)
    def __getitem__(self, name): return getattr(self,name,None)
    def __getattr__(self,key): return None
    def has_key(self,key): return True if self.__dict__.has_key(key) else False
    def keys(self): return list(self.__dict__)

if __name__ == "__main__":
    #print "** Starting..."
    sys.exit(main(sys.argv))