# coding: utf-8
#
# This file is part of GetPicons - enigma2 plugin to download picons from lyngsat.com
# Copyright (c) 2016 ivars777 (ivars777@gmail.com)
# Distributed under the GNU GPL v3. For full terms see http://www.gnu.org/licenses/gpl-3.0.en.html
"""
Dowload and create Enigma2 channels' picons files form lyngsat.com
Usage [options]
Options:
-p PACKAGE_LIST, --package=PACKAGE_LIST list of package names (html file name in lyngsat),
e.g "viasat,ntvplus36"
-s SAT_LIST, --sat=SAT_LIST list of sattelite positions, e.g. "4.9,-0.8"
-z SIZE icon size, e.g. "100x60", default - "220x132"
-f PATH, --folder PATH picon files output folder, default - "/media/hdd/picon"
-e PATH, --enigma PATH enigma2 folder whera lamedb,settings are located, default - "/etc/enigma2"
you can use urls, e.g. "ftp://root@receiver_address/etc/enigma2"
-o, -overwrite overwrite existing picons file (default - no)
-d, -debug display work progress and write debug info to file for not found services
-h, --help this help file
(c)Ivars777 (ivars777@gmail.com) 2013-2015, v0.2
"""
import sys, os, os.path, getopt
import re
import urllib2
import requests
try:
import Image
except:
from PIL import Image
from StringIO import StringIO
import logging
options = args = None
_sd= lambda x,d: d if x is None else x # set default value
_sl= lambda x: False if x is None else True # set True of False for option
def parse_arguments(argv,opt_short,opt_long):
"Parse command line arguments"
try:
options, args = getopt.gnu_getopt(argv, opt_short,opt_long)
options = Rec(dict([(o[0].replace("-",""),o[1]) for o in options]))
except getopt.GetoptError, err:
print err.msg
print __doc__
sys.exit(2)
if options.has_key("h") or options.has_key("help"):
print __doc__
sys.exit(2)
return options,args
services = None
def main(argv):
global options, args, services
# Parsing options
opt_short = 'p:s:z:f:e:odh'
opt_long = ["package=","sat","folder=","enigma=","overwrite","debug","help"]
options,args = parse_arguments(argv[1:], opt_short, opt_long)
options.package = _sd(options.package,_sd(options.p,""))
options.sat = _sd(options.sat,_sd(options.s,""))
options.enigma = _sd(options.enigma,_sd(options.e,"/etc/enigma2"))
options.folder = _sd(options.folder,_sd(options.f,"/media/hdd/picon"))
options.size = _sd(options.size,_sd(options.z,"220x132"))
options.overwrite = _sl(_sd(options.overwrite,options.o))
options.debug = _sl(_sd(options.debug,options.d))
options.w,options.h = map(int,options.size.split("x"))
if not os.path.exists(options.folder):
os.makedirs(options.folder)
if options.debug:
#filename="get_picons.log" if not os.name == "posix" else "/var/log/get_picons.log"
filename = os.path.join(options.folder,"get_picons.log")
FORMAT = "%(asctime)-15s %(message)s"
logging.basicConfig(filename=filename, format=FORMAT,level=logging.DEBUG)
print "** Analyzing lamedb"
services = DBServices(options.enigma)
if options.sat:
package_list = []
for sat in options.sat.split(","):
package_list.extend(get_packages(sat))
elif options.package:
package_list = options.package.split(",")
else: # get sattelite list from Enigma settings
sat_list = []
package_list = []
if not "ftp:" in options.enigma:
fname = os.path.join(options.enigma,"settings")
f = open(fname)
else:
sep = "" if options.enigma[-1]=="//" else "//"
fname = options.enigma + sep + "settings"
f = urllib2.urlopen(fname)
for line in f:
if not "config.Nims.0.advanced.sat." in line: continue
sat_list.append(line.split(".")[5])
for sat in sat_list:
package_list.extend(get_packages(sat))
for package in package_list:
create_picons(package)
def create_picons(package):
"Create picons files for package"
global services,options
url_package = "http://www.lyngsat.com/packages/%s.html"%package
if options.debug:
print "** Downloading and looking %s for picons"%url_package
num_picons = 0
num_skipped = 0
num = 0
try:
#soup = html = urllib2.urlopen(url_package).read().decode("latin1")
soup = html = get_page(url_package).decode("latin1")
except:
raise Exception("Package page '%s' does not found"%url_package)
title = re.search("(.+?)", soup, re.DOTALL | re.IGNORECASE).group(1)
pos = re.search("\d+\.\d+",title).group().replace(".","")
ew = re.search("\xb0.",title).group()[1]
ns = int(pos)
if ew == "W":
ns = 3600 - ns
if ns==48: ns=49
if ns==359: ns=360
#pos = "%04x0000"%ns
#pos_num = int(pos[0:4],16)
title2 = title[0:title.index("- LyngSat")]
print "** %s: "%title2.replace(u'\xb0',""),
tables = re.findall(r"", soup, re.DOTALL | re.IGNORECASE)
for table in tables:
tr= html_findall("tr",table)
for i in range(2,len(tr)-1):
td= html_findall("td",tr[i])
if len(td) == 10:
freq,polar = re.search(r"(\d+) ([HVLR])", tr[i], re.DOTALL | re.IGNORECASE).groups()
freq = int(freq)
p_value = {"H":0,"V":1,"L":2,"R":3}
polar = p_value[polar]
b=1
else:
b=0
icon_url = html_attr("src",td[b])
if icon_url:
icon_url = "http://www.lyngsat.com" + icon_url.group(1).replace("/icon","/logo").replace(".gif",".jpg")
else:
icon_url = ""
name = html_text(td[b+1]).group(1)
sid = html_text(td[b+5])
sid = sid.group(1).replace(" ","") if sid else ""
sid = int(sid) if sid and not sid == u"-" else ""
vpid = html_text(td[b+6])
vpid = vpid.group(1).replace(" ","") if vpid else ""
#try:
# vpid = int(vpid)
#except Exception:
# vpid = None
#vpid = int(vpid) if vpid and not vpid == u"-" else None
if not sid or not vpid or icon_url=="":
continue
if options.debug:
print " package:%i/%s: service:%s "%(ns,package, name.encode("ascii","replace")),
num += 1
sref = find_sref(ns,freq,polar,sid)
if not sref:
sref = find_sref(ns,freq+1,polar,sid)
if not sref:
sref = find_sref(ns,freq-1,polar,sid)
if not sref:
if options.debug:
print " -- NOK (no sref in lamedb!)"
logging.debug(u" no sref - package:%i/%s: service:%s pos:%s/%s freq:%i polar:%s sid:%i/%x"%(ns,package, name,ns, pos_int2str(ns),freq,polar,sid,sid))
continue
fname = sref.replace(":","_")
fname = fname[:-1]+".png"
fname = os.path.join(options.folder, fname)
if os.path.exists(fname) and not options.overwrite:
if options.debug: print " -- skipped"
num_skipped += 1
continue
#try:
#im = Image.open(StringIO(urllib2.urlopen(icon_url).read()))
data = get_page(icon_url)
im = Image.open(StringIO(data))
#im = im.crop((0,10,132,89))
im = im.resize((options.w,options.h), Image.ANTIALIAS)
im = im.convert('P', palette=Image.ADAPTIVE)
#except Exception:
# im = None
if not im:
if options.debug: print " -- NOK (no picon image)"
continue
if options.debug: print " -- downloaded"
im.save(fname,"png")
num_picons += 1
print "%i picons created, %i skipped"%(num_picons,num_skipped)
def get_packages(sat):
"Get packages list (names) for sattelite from lyngsat.com page"
global options
sat= sat.replace(".","")
num = int(sat)
if num>1800: num = num - 3600
if num>0 and num<730: soup = get_soup("europe")
elif num>730 and num<1600: soup = get_soup("asia")
elif num<0 and num>-610: soup = get_soup("atlantic")
elif num<-610 and num>-1600: soup = get_soup("america")
else:
return []
i1,i2 = gr=re.search(r"", soup, re.DOTALL | re.IGNORECASE).span()
packages = []
for s in re.findall("(.+?)
", soup[i1:i2], re.DOTALL | re.IGNORECASE):
gr = re.search(r"(\d+.)(\d).+#176;([EW])", s,re.DOTALL | re.IGNORECASE).groups()
pos = int("".join(gr[:-1]).replace(".",""))
if gr[2] == "W": pos = -pos
if abs(num - pos) <=2:
package = re.search(r"www\.lyngsat\.com/packages/(\w.+?)\.html", s, re.DOTALL | re.IGNORECASE).group(1)
packages.append(package)
return packages
soups = {}
def get_soup(region):
if soups.has_key(region):
return soups[region]
url = "http://www.lyngsat.com/packages/%s.html"%region
if options.debug:
print "** Downloading and looking %s for packages"%url
try:
#html = urllib2.urlopen(url).read().decode("latin1")
html = get_page(url).decode("latin1")
except:
raise Exception("Packege page '%s' does not found"%url)
soups[region] = html
return soups[region]
def find_sref(ns,freq,polar,sid):
"Find service reference according to given parameters"
global services
tp = services.get_transp_by_pic((ns,freq,polar))
if tp:
serv = services.get_service_by_chid((sid,tp["tsid"],tp["onid"]))
if serv:
return serv["sref"]
class DBServices(object):
"""Dreambox services, stored in lamedb (E2)/services(E1) file handling class"""
def __init__(self,service_path,enigma=2):
self.enigma = enigma
self.service_path = service_path
if self.enigma == 2:
if not "ftp:" in service_path:
self.service_fname= os.path.join(service_path,"lamedb")
self.bouquets_fname = os.path.join(service_path, "bouquets.tv")
else:
sep = "" if service_path[-1]=="//" else "//"
self.service_fname= service_path+sep+"lamedb"
self.bouquets_fname = service_path+sep+"bouquets.tv"
else: # ENIGMA === "1":
self.service_fname = os.path.join(service_path,"services")
self.bouquets_fname = os.path.join(service_path, "userbouquets.tv.epl")
self.services = []
self.index_sref= {}
self.index_chid = {}
self.index_name = {}
self.index_provider = {}
self.bouquets = []
self.bouquets_services = {}
self.transp = []
self.tp_index_tpid = {}
self.tp_index_pic ={}
self._load_services()
#self._load_bouquets()
def _load_services(self):
#tpref - lamedb/transponders
# NS:TSID:ONID
# s FREQ:SR:POLAR:FREC:49:2:0
# X X X
# D D D D
# lref - lamedb/services
# SID:NS:TSID:ONID:STYPE:UNUSED(channelnumber in enigma1)
# 0 1 2 3 4 5
# X X X X D D
# wref bouquets/picon
# REFTYPE:FLAGS:STYPE:SID:TSID:ONID:NS:PARENT_SID:PARENT_TSID:UNUSED
# 0 1 2 3 4 5 6 7 8 9
# D D X X X X X X X X
f_services = open(self.service_fname,"r") if not self.service_fname[0:4] == "ftp:" else urllib2.urlopen(self.service_fname)
line = f_services.readline()
if not "eDVB services" in line:
raise Exception("no correct lamedb file")
line = f_services.readline()
# Read transponders
i = 0
while True:
line = f_services.readline()
if "end" in line: break
#tp = record.Rec()
tp = {}
ff = line.strip().split(":")
tp["ns"] = ns = pos_str2int(ff[0])
tp["tsid"] = tsid = int(ff[1],16)
tp["onid"] = onid = int(ff[2],16)
line = f_services.readline()
tp["params"] = params = line.strip().split(" ")[1]
ff = params.split(":")
tp["freq"] = freq = int(ff[0][:-3])
tp["polar"] = polar = int(ff[2])
tp["tpid"] = tpid = (ns,tsid,onid)
self.transp.append(tp)
self.tp_index_tpid[tpid] = i
self.tp_index_pic[(ns,freq,polar)] = i
line = f_services.readline()
i += 1
# Reading services
i= 0
line = f_services.readline()
while True:
line = f_services.readline()
if line[0:3] == "end": break
if not line: break
#rec = record.Rec()
rec = {}
rec["lref"] = line.lower().strip().decode("utf-8")
line = f_services.readline()
#line = line.replace('\xc2\x87', '').replace('\xc2\x86', '')
#line = decode_charset(line)
line = line.decode("utf-8")
rec["name"] = line.strip()
line = f_services.readline()
pl = line.split(",")
provider = ""
for p in pl:
if p[0:2] == "p:":
provider = p[2:].strip()
break
if provider == "":
provider = "unknown"
rec["provider"] = provider.decode("utf-8")
rec["sref"] = lref2sref(rec["lref"])
r = lref_parse(rec["lref"])
rec["stype"] = r["stype"]
rec["chid"] = (r["sid"],r["tsid"],r["onid"])
self.services.append(rec)
self.index_sref[rec["sref"]] = i
self.index_chid[rec["chid"]] = i
name = rec["name"].lower()
if not self.index_name.has_key(name): self.index_name[name] = []
self.index_name[name].append(i)
provider = rec["provider"].lower()
if not self.index_provider.has_key(provider): self.index_provider[provider] = []
self.index_provider[provider].append(i)
i += 1
f_services.close()
def _load_bouquets(self):
f_bouquets = open(self.bouquets_fname,"r") if not self.bouquets_fname[0:4] == "ftp:" else urllib2.urlopen(self.bouquets_fname)
for line in f_bouquets.readlines():
if line[0:9] == "#SERVICE:":
if self.enigma==1:
bn = line.strip().split("/")
else:
bn = line.strip().split(":")
bfn = bn[-1]
bf = open(os.path.join(self.service_path,bfn))
for line2 in bf.readlines():
if line2[0:5] == "#NAME":
bname = line2.strip()[6:]
self.bouquets.append(bname)
self.bouquets_services[bname] = []
elif line2[0:8] == "#SERVICE":
sref = line2.strip()[9:]
r = sref_parse(sref)
if r["flags"] == 64: continue
self.bouquets_services[bname].append(sref)
bf.close()
f_bouquets.close()
def get_bouquets(self):
if not self.bouquets:
self._load_bouquets()
return self.bouquets
def get_bouquet_services(self,bn):
if not self.bouquets:
self._load_bouquets()
return [self.get_service_by_sref(sref) for sref in self.bouquets_services[bn]]
def get_service_by_sref(self,sref):
return self.services[self.index_sref[sref]] if self.index_sref.has_key(sref) else None
def get_service_by_chid(self,chid):
return self.services[self.index_chid[chid]] if self.index_chid.has_key(chid) else None
def get_service_by_name(self,name):
return [self.services[i] for i in self.index_name[name]]
def get_service_by_provider(self,provider):
return [self.services[i] for i in self.index_provider[provider]]
def get_transp_by_tpid(self,tpid):
return self.transp[self.tp_index_tpid[tpid]] if self.tp_index_tpid.has_key(tpid) else None
def get_transp_by_pic(self,picid):
return self.transp[self.tp_index_pic[picid]] if self.tp_index_pic.has_key(picid) else None
def decode_charset(s):
u = None
if isinstance(s,unicode):
return s
charset_list=('iso-8859-4','iso-8859-1','iso-8859-2''iso-8859-15')
for charset in charset_list:
try:
u=unicode(s,charset,"strict")
except:
pass
else:
break
if u == None:
raise Error, "CHARSET ERROR while decoding lamedb. Aborting !"
else:
return(u)
# lref - lamedb/services
# SID:NS:TSID:ONID:STYPE:UNUSED(channelnumber in enigma1)
# 0 1 2 3 4 5
# X X X X D D
#
# wref bouquets/picon
# REFTYPE:FLAGS:STYPE:SID:TSID:ONID:NS:PARENT_SID:PARENT_TSID:UNUSED
# 0 1 2 3 4 5 6 7 8 9
# D D X X X X X X X X
def lref2sref(s):
"Converts service refefence from lamedb format to bouquets/picon (standard) format"
f = s.split(":")
sid = int(f[0],16)
ns = int(f[1], 16)
tsid = int(f[2],16)
onid = int(f[3],16)
stype = int(f[4])
s2 = "1:0:%X:%X:%X:%X:%X:0:0:0:" % (stype,sid,tsid,onid,ns)
return s2
def lref_parse(s):
"Parse lamedb/service string, return dictionary of items"
f = s.split(":")
r = {}
r["sid"] = int(f[0],16)
r["ns"] = int(f[1], 16)
r["tsid"] = int(f[2],16)
r["onid"] = int(f[3],16)
r["stype"] = int(f[4])
return r
def sref2lref(s):
"Converts service refefence from bouquets/picon (standard) format to lamedb format"
f = s.split(":")
reftype = int(f[0])
flags = int(f[1])
stype = int(f[2],16)
sid = int(f[3],16)
tsid = int(f[4],16)
onid = int(f[5],16)
ns = int(f[6],16)
s2 = "%04x:%08x:%04x:%04x:%i:0" % (sid,ns,tsid,onid,stype)
return s2
def sref_parse(s):
"Parse service refefence string , return dictionary of items"
f = s.split(":")
r = {}
r["reftype"] = int(f[0])
r["flags"] = int(f[1])
r["stype"] = int(f[2],16)
r["sid"] = int(f[3],16)
r["tsid"] = int(f[4],16)
r["onid"] = int(f[5],16)
r["ns"] = int(f[6],16)
return r
pos_int2str = lambda pos_num: "%04x0000"%pos_num
pos_str2int = lambda pos: int(pos[0:4],16)
def decode_charset(s):
u = None
charset_list = ('utf-8','iso-8859-1','iso-8859-2','iso-8859-15')
for charset in charset_list:
try:
u = unicode(s,charset,"strict")
except:
pass
else:
break
if u == None:
print("CHARSET ERROR while decoding string")
sys.exit(1)
else:
return(u)
headers2dict = lambda h: dict([l.strip().split(": ") for l in h.strip().splitlines()])
headers = headers2dict("""
User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0
Referer: http://www.lyngsat.com/
""")
def get_page(url):
r=requests.get(url,headers=headers)
return r.content
html_text = lambda html: re.search(r">([^<^\n]+?)<", html, re.DOTALL | re.IGNORECASE)
html_tag = lambda tag, html: re.search(r"<%s\b[^>]*>(.*?)"%(tag,tag), html, re.DOTALL | re.IGNORECASE)
html_attr = lambda attr,html: re.search("""%s="?([^'" >]+)"""%(attr), html, re.DOTALL | re.IGNORECASE)
html_findall_content = lambda tag,html: re.findall(r"<%s\b[^>]*>(.*?)"%(tag,tag), html, re.DOTALL | re.IGNORECASE)
html_findall = lambda tag,html: re.findall(r"<%s\b[^>]*>.*?"%(tag,tag), html, re.DOTALL | re.IGNORECASE)
import UserDict
class Rec(UserDict.DictMixin):
"Dict like class allowing a.x instead of a['x']"
def __init__(self, dict=None, **kwargs):
self.__dict__ = {}
if dict is not None:
self.update(dict)
if len(kwargs):
self.update(kwargs)
def __setitem__(self, name, value): setattr(self, name, value)
def __getitem__(self, name): return getattr(self,name,None)
def __getattr__(self,key): return None
def has_key(self,key): return True if self.__dict__.has_key(key) else False
def keys(self): return list(self.__dict__)
if __name__ == "__main__":
#print "** Starting..."
sys.exit(main(sys.argv))