Python module (submodule repositary), which provides content (video streams) from various online stream sources to corresponding Enigma2, Kodi, Plex plugins

util.py 24KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766
  1. # -*- coding: UTF-8 -*-
  2. # /*
  3. # * Copyright (C) 2011 Libor Zoubek,ivars777
  4. # *
  5. # *
  6. # * This Program is free software; you can redistribute it and/or modify
  7. # * it under the terms of the GNU General Public License as published by
  8. # * the Free Software Foundation; either version 2, or (at your option)
  9. # * any later version.
  10. # *
  11. # * This Program is distributed in the hope that it will be useful,
  12. # * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # * GNU General Public License for more details.
  15. # *
  16. # * You should have received a copy of the GNU General Public License
  17. # * along with this program; see the file COPYING. If not, write to
  18. # * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  19. # * http://www.gnu.org/copyleft/gpl.html
  20. # *
  21. # */
  22. import os, sys, re
  23. import urllib, urllib2
  24. import datetime, time
  25. import traceback
  26. import cookielib
  27. import requests
  28. try:
  29. from requests.packages.urllib3.exceptions import InsecureRequestWarning
  30. requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
  31. except:
  32. pass
  33. from htmlentitydefs import name2codepoint as n2cp
  34. import HTMLParser
  35. import StringIO
  36. #import threading
  37. #import Queue
  38. import pickle
  39. import string
  40. import simplejson as json
  41. #from demjson import demjson
  42. #import demjson
  43. import json
  44. #from bs4 import BeautifulSoup
  45. UA = 'Mozilla/6.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.5) Gecko/2008092417 Firefox/3.0.3'
  46. LOG = 2
  47. _cookie_jar = None
  48. CACHE_COOKIES = 'cookies'
  49. def system():
  50. if "kodi" in sys.executable.lower():
  51. return "kodi"
  52. elif sys.platform == "win32":
  53. return "windows"
  54. elif sys.platform == "linux2":
  55. return "enigma2"
  56. else:
  57. return "unknown"
  58. def nfo2xml(nfo_dict):
  59. nfo_type,nfo = next(nfo_dict.iteritems())
  60. s= "<%s>\n"%nfo_type.encode("utf8")
  61. for k,v in nfo.iteritems():
  62. if isinstance(v,list):
  63. for v2 in v:
  64. if isinstance(v2,unicode): v2 = v2.encode("utf8")
  65. s += " <%s>%s</%s>\n"%(k.encode("utf8"), v2, k.encode("utf8"))
  66. else:
  67. if isinstance(v,unicode): v = v.encode("utf8")
  68. s += " <%s>%s</%s>\n"%(k.encode("utf8"), v, k.encode("utf8"))
  69. s += "</%s>\n"%nfo_type.encode("utf8")
  70. return s
  71. def nfo2desc(nfo):
  72. if not "title" in nfo:
  73. nfo_type, nfo = next(nfo.iteritems())
  74. desc = nfo2title(nfo)
  75. dd = lambda t: "\n" + nfo[t] if t in nfo and nfo[t] else ""
  76. dd2 = lambda t: "\n" + ",".join(nfo[t]) if t in nfo and nfo[t] else ""
  77. def ddd(t,title=""):
  78. if title:
  79. title = title + ": "
  80. if t in nfo and nfo[t]:
  81. if isinstance(nfo[t],list):
  82. d = "\n" + title + ",".join(nfo[t])
  83. else:
  84. d = "\n" + title + nfo[t]
  85. else:
  86. d = ""
  87. return d.encode("utf8") if isinstance(d, unicode) else d
  88. desc += ddd("genre","Genre")
  89. desc += ddd("runtime","Length")
  90. desc += ddd("director","Director")
  91. desc += ddd("actor","Actors")
  92. desc += ddd("language","Languages")
  93. desc += ddd("quality","Quality")
  94. desc += ddd("tagline")
  95. if "plot" in nfo and "tagline" in nfo and nfo["tagline"] <> nfo["plot"]:
  96. desc += ddd("plot")
  97. elif "plot" in nfo and not "tagline" in nfo:
  98. desc += ddd("plot")
  99. return desc.encode("utf8") if isinstance(desc,unicode) else desc
  100. def nfo2title(nfo):
  101. if not "title" in nfo:
  102. nfo_type, nfo = next(nfo.iteritems())
  103. title = nfo["title"]
  104. if "originaltitle" in nfo and nfo["originaltitle"] and nfo["originaltitle"]<>nfo["title"]:
  105. title +=" ~ "+nfo["originaltitle"]
  106. se = ""
  107. if "season" in nfo and nfo["season"]:
  108. se += "S%s" % nfo["season"]
  109. if "episode" in nfo and nfo["episode"]:
  110. se += "E%s" % nfo["episode"]
  111. if se:
  112. title += " (%s)" % se
  113. elif "year" in nfo and nfo["year"]:
  114. title += " (%s)"%nfo["year"]
  115. return title.encode("utf8") if isinstance(title,unicode) else title
  116. def stream_type(data):
  117. data = data.lower()
  118. m = re.search(r"^(\w+)://", data)
  119. prefix = m.group(1) if m else ""
  120. if prefix in ("http","https"):
  121. if ".m3u8" in data:
  122. return "hls"
  123. elif ".mpd" in data:
  124. return "dash"
  125. else:
  126. return "http"
  127. else:
  128. return prefix
  129. def check_version(package,url="http://feed.blue.lv/Packages"):
  130. "Return current package version from OPKG feed"
  131. url = "http://feed.blue.lv/Packages"
  132. r = requests.get(url)
  133. if not r.ok:
  134. return ""
  135. m = re.search("Package: %s\nVersion: (.+?)\n"%package, r.content)
  136. if not m:
  137. return ""
  138. return m.group(1)
  139. SPLIT_CHAR = "~"
  140. SPLIT_CODE = urllib.quote(SPLIT_CHAR)
  141. EQ_CODE = urllib.quote("=")
  142. COL_CODE = urllib.quote(":")
  143. SPACE_CODE = urllib.quote(" ")
  144. def make_fname(title):
  145. "Make file name from title"
  146. title = title.strip()
  147. fname0 = re.sub("[/\n\r\t,:]"," ",title)
  148. fname0 = re.sub("['""]","",fname0)
  149. return fname0
  150. def hls_base(url):
  151. url2 = url.split("?")[0]
  152. url2 = "/".join(url2.split("/")[0:-1])+ "/"
  153. return url2
  154. def stream_change(stream):
  155. #return stream # TODO
  156. if "resolver" in stream and stream["resolver"] in ("viaplay","hqq","filmas") or \
  157. "surl" in stream and re.search("https*://(hqq|goo\.gl)",stream["surl"]):
  158. stream["url"] = streamproxy_encode(stream["url"],stream["headers"])
  159. stream["headers"] = {}
  160. return stream
  161. else:
  162. return stream
  163. def streamproxy_encode(url,headers=[],proxy_url=None):
  164. PROXY_URL = "http://localhost:8880/"
  165. if not "?" in url:
  166. url = url+"?"
  167. url2 = url.replace(SPLIT_CHAR,SPLIT_CODE).replace(":",COL_CODE).replace(" ",SPACE_CODE)
  168. if not proxy_url:
  169. proxy_url = PROXY_URL
  170. url2 = proxy_url + url2
  171. if headers:
  172. headers2 = []
  173. for h in headers:
  174. headers2.append("%s=%s"%(h,headers[h].replace("=",EQ_CODE).replace(SPLIT_CHAR,SPLIT_CODE).replace(" ",SPACE_CODE)))
  175. headers2 = SPLIT_CHAR.join(headers2)
  176. url2 = url2+SPLIT_CHAR+headers2
  177. #return url2.encode("utf8") if isinstance(url2,unicode) else url2
  178. return url2
  179. def streamproxy_decode(urlp):
  180. import urlparse
  181. path = urlp.replace(re.search("http://[^/]+",urlp).group(0),"")
  182. p = path.split(SPLIT_CHAR)
  183. url = urllib.unquote(p[0][1:])
  184. #headers = {"User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 9_2 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/47.0.2526.70 Mobile/13C71 Safari/601.1.46"}
  185. headers={}
  186. if len(p)>1:
  187. for h in p[1:]:
  188. #h = urllib.unquote()
  189. headers[h.split("=")[0]]=urllib.unquote(h.split("=")[1])
  190. return url,headers
  191. def streamproxy_encode2(url,headers=[],proxy_url=None):
  192. PROXY_URL = "http://localhost:8880/"
  193. #url2 = url.replace(SPLIT_CHAR,SPLIT_CODE).replace(":",COL_CODE).replace(" ",SPACE_CODE)
  194. url2 = urllib.quote_plus(url)
  195. if not proxy_url:
  196. proxy_url = PROXY_URL
  197. url2 = proxy_url + url2+"/?"
  198. if headers:
  199. headers2 = []
  200. for h in headers:
  201. headers2.append("%s=%s"%(h,headers[h].replace("=",EQ_CODE).replace(SPLIT_CHAR,SPLIT_CODE).replace(" ",SPACE_CODE)))
  202. headers2 = SPLIT_CHAR.join(headers2)
  203. url2 = url2+SPLIT_CHAR+headers2
  204. return url2
  205. def streamproxy_decode2(urlp):
  206. path = urlp.replace(re.search("http://[^/]+",urlp).group(0),"")
  207. p = path.split(SPLIT_CHAR)
  208. url = urllib.unquote_plus(p[0][1:-2])
  209. #headers = {"User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 9_2 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/47.0.2526.70 Mobile/13C71 Safari/601.1.46"}
  210. headers={}
  211. if len(p)>1:
  212. for h in p[1:]:
  213. #h = urllib.unquote()
  214. headers[h.split("=")[0]]=urllib.unquote(h.split("=")[1])
  215. return url,headers
  216. def streamproxy_encode3(cmd, data, proxy_url=None, headers=None, qs=None):
  217. PROXY_URL = "http://localhost:8880/"
  218. #url2 = url.replace(SPLIT_CHAR,SPLIT_CODE).replace(":",COL_CODE).replace(" ",SPACE_CODE)
  219. data2 = urllib.quote_plus(data)
  220. if not proxy_url:
  221. proxy_url = PROXY_URL
  222. urlp = proxy_url + cmd + "/" + data2 + "/"
  223. qs2 = []
  224. if headers:
  225. headers2 = []
  226. for h in headers:
  227. headers2.append("%s=%s"%(h,urllib.quote_plus(headers[h])))
  228. headers2 = SPLIT_CHAR.join(headers2)
  229. qs2.append("headers="+urllib.quote_plus(headers2))
  230. if qs:
  231. for qv in qs:
  232. qs2.append(qv+"="+urllib.quote_plus(qs[qv]))
  233. if qs2:
  234. qs2 = "&".join(qs2)
  235. if qs2:
  236. urlp = urlp + "?" + qs2
  237. return urlp
  238. def streamproxy_decode3(urlp):
  239. m = re.search("http://[^/]+",urlp)
  240. urlp = urlp.replace(m.group(0),"") if m else urlp
  241. path = urlp.split("?")[0]
  242. cmd = path.split("/")[1]
  243. data = "/".join(path.split("/")[2:])
  244. data = urllib.unquote_plus(data)
  245. qs = urlp.split("?")[1] if "?" in urlp else ""
  246. qs2 = {}
  247. headers = {}
  248. if qs:
  249. qs = qs.split("&")
  250. for q in qs:
  251. qk = q.split("=")[0]
  252. if qk == "headers":
  253. hh = urllib.unquote_plus(q.split("=")[1])
  254. hh = hh.split(SPLIT_CHAR)
  255. for h in hh:
  256. headers[h.split("=")[0]] = urllib.unquote_plus(h.split("=")[1])
  257. else:
  258. qv = urllib.unquote_plus(q.split("=")[1]) if "=" in q else ""
  259. qs2[qk] = qv
  260. return cmd, data, headers, qs2
  261. class Captions(object):
  262. def __init__(self,uri):
  263. self.uri = uri
  264. self.subs = []
  265. self.styles = {}
  266. if uri.startswith("http"):
  267. r = requests.get(uri)
  268. if r.status_code == 200:
  269. self.loads(r.content)
  270. def loads(self,s):
  271. if "WEBVTT" in s[:s.find("\n")]: # vtt captions
  272. self.load_vtt(s)
  273. elif "<?xml" in s[:s.find("\n")]:
  274. self.load_ttaf(s)
  275. else:
  276. self.load_vtt(s) # TODO
  277. def load_ttaf(self,s):
  278. for r2 in re.findall("<style .+?/>", s):
  279. st = {}
  280. for a in re.findall(r'(\w+)="([^ "]+)"', r2):
  281. st[a[0]] = a[1]
  282. if a[0] == "id":
  283. sid = a[1]
  284. self.styles[sid] = st
  285. for r2 in re.findall("<p .+?</p>", s):
  286. sub = {}
  287. sub["begin"] = str2sec(re.search('begin="([^"]+)"', r2).group(1)) if re.search('begin="([^"]+)"', r2) else -1
  288. sub["end"] = str2sec(re.search('end="([^"]+)"', r2).group(1)) if re.search('end="([^"]+)"', r2) else -1
  289. sub["style"] = re.search('style="([^"]+)"', r2).group(1) if re.search('style="([^"]+)"', r2) else None
  290. sub["text"] = re.search("<p[^>]+>(.+)</p>", r2).group(1).replace("\n","")
  291. sub["text"] = re.sub("<br\s*?/>","\n",sub["text"])
  292. sub["text"] = re.sub("<.+?>"," ",sub["text"])
  293. self.subs.append(sub)
  294. pass
  295. def load_vtt(self,s):
  296. f = StringIO.StringIO(s)
  297. while True:
  298. line = f.readline()
  299. if not line:
  300. break
  301. m = re.search(r"([\d\.\,:]+)\s*-->\s*([\d\.\,\:]+)",line)
  302. if m:
  303. sub = {}
  304. sub["begin"] = str2sec(m.group(1))
  305. sub["end"] = str2sec(m.group(2))
  306. sub["style"] = None
  307. sub["text"] = []
  308. line = f.readline()
  309. while line.strip():
  310. txt = line.strip()
  311. if isinstance(txt,unicode):
  312. txt = txt.encode("utf8")
  313. sub["text"].append(txt)
  314. line = f.readline()
  315. sub["text"] = "\n".join(sub["text"])
  316. self.subs.append(sub)
  317. else:
  318. continue
  319. pass
  320. def get_srt(self):
  321. out = ""
  322. i = 0
  323. for sub in self.subs:
  324. i +=1
  325. begin = sub["begin"]
  326. begin = "%s,%03i"%(str(datetime.timedelta(seconds=begin/1000)),begin%1000)
  327. end = sub["end"]
  328. end = "%s,%03i"%(str(datetime.timedelta(seconds=end/1000)),end%1000)
  329. txt2 = sub["text"]
  330. out += "%s\n%s --> %s\n%s\n\n\n"%(i,begin,end,txt2)
  331. return out
  332. def str2sec(r):
  333. # Convert str time to miliseconds
  334. r= r.replace(",",".")
  335. m = re.search(r"(\d+\:)*(\d+)\:(\d+\.\d+)", r)
  336. if m:
  337. sec = int(m.group(1)[:-1])*60*60*1000 if m.group(1) else 0
  338. sec += int(m.group(2))*60*1000 + int(float(m.group(3))*1000)
  339. return sec
  340. else:
  341. return -1
  342. #c = Captions("http://195.13.216.2/mobile-vod/mp4:lb_barbecue_fr_lq.mp4/lb_barbecue_lv.vtt")
  343. #c = Captions("http://www.bbc.co.uk/iplayer/subtitles/ng/modav/bUnknown-0edd6227-0f38-411c-8d46-fa033c4c61c1_b05ql1s3_1479853893356.xml")
  344. #url = "http://195.13.216.2/mobile-vod/mp4:ac_now_you_see_me_2_en_lq.mp4/ac_now_you_see_me_2_lv.vtt"
  345. #c = Captions(url)
  346. #pass
  347. def ttaf2srt(s):
  348. out = u""
  349. i = 0
  350. for p,txt in re.findall("<p ([^>]+)>(.+?)</p>", s, re.DOTALL):
  351. i +=1
  352. begin = re.search('begin="(.+?)"',p).group(1)
  353. begin = begin.replace(".",",")
  354. end = re.search('end="(.+?)"',p).group(1)
  355. end = end.replace(".",",")
  356. txt2 = re.sub("<br */>","\n",txt)
  357. out += "%s\n%s --> %s\n%s\n\n"%(i,begin,end,txt2)
  358. return out
  359. def vtt2srt(s):
  360. out = ""
  361. i = 0
  362. # result = re.findall(r"([\d:\. ]+-->[\d:\. ]+)\n(.+?\n)+", r.content)
  363. for t1, t2, txt in re.findall(r"([\d:\.]+)\s*-->\s*([\d:\.]+)\n(.+?\n)+", s):
  364. i +=1
  365. if len(t1.split(":")) == 2:
  366. t1 = "00:" + t1
  367. t1 = t1.replace(".", ",")
  368. if len(t2.split(":")) == 2:
  369. t2 = "00:" + t2
  370. t2 = t2.replace(".", ",")
  371. out += "%s\n%s --> %s\n%s\n\n"%(i,t1,t2,txt)
  372. return out
  373. def item():
  374. """Default item content"""
  375. stream0 = {
  376. 'name': '', #
  377. 'url': '',
  378. 'quality': '?',
  379. 'surl': '',
  380. 'subs': [],
  381. 'headers': {},
  382. "desc":"",
  383. "img":"",
  384. "lang":"",
  385. "type":"",
  386. "resolver":"",
  387. "order":0,
  388. "live":False
  389. }
  390. return stream0
  391. class _StringCookieJar(cookielib.LWPCookieJar):
  392. def __init__(self, string=None, filename=None, delayload=False, policy=None):
  393. cookielib.LWPCookieJar.__init__(self, filename, delayload, policy)
  394. if string and len(string) > 0:
  395. self._cookies = pickle.loads(str(string))
  396. def dump(self):
  397. return pickle.dumps(self._cookies)
  398. def init_urllib(cache=None):
  399. """
  400. Initializes urllib cookie handler
  401. """
  402. global _cookie_jar
  403. data = None
  404. if cache is not None:
  405. data = cache.get(CACHE_COOKIES)
  406. _cookie_jar = _StringCookieJar(data)
  407. opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(_cookie_jar))
  408. urllib2.install_opener(opener)
  409. def cache_cookies(cache):
  410. """
  411. Saves cookies to cache
  412. """
  413. global _cookie_jar
  414. if _cookie_jar:
  415. cache.set(CACHE_COOKIES, _cookie_jar.dump())
  416. def request0(url, headers={}):
  417. debug('request: %s' % url)
  418. req = urllib2.Request(url, headers=headers)
  419. req.add_header('User-Agent', UA)
  420. try:
  421. response = urllib2.urlopen(req)
  422. data = response.read()
  423. response.close()
  424. except urllib2.HTTPError, error:
  425. data = error.read()
  426. debug('len(data) %s' % len(data))
  427. return data
  428. def request(url, headers={}):
  429. debug('request: %s' % url)
  430. #req = urllib2.Request(url, headers=headers)
  431. #req.add_header('User-Agent', UA)
  432. if 'User-Agent' not in headers:
  433. headers['User-Agent']= UA
  434. try:
  435. r = requests.get(url, headers=headers)
  436. data = r.content
  437. except:
  438. data = r.content
  439. debug('len(data) %s' % len(data))
  440. return data
  441. def post(url, data, headers={}):
  442. postdata = urllib.urlencode(data)
  443. #req = urllib2.Request(url, postdata, headers)
  444. #req.add_header('User-Agent', UA)
  445. import requests
  446. if 'User-Agent' not in headers:
  447. headers['User-Agent']= UA
  448. try:
  449. r = requests.post(url, data=postdata,headers=headers)
  450. data = r.content
  451. except urllib2.HTTPError, error:
  452. data = r.content
  453. return data
  454. def post0(url, data, headers={}):
  455. postdata = urllib.urlencode(data)
  456. req = urllib2.Request(url, postdata, headers)
  457. req.add_header('User-Agent', UA)
  458. try:
  459. response = urllib2.urlopen(req)
  460. data = response.read()
  461. response.close()
  462. except urllib2.HTTPError, error:
  463. data = error.read()
  464. return data
  465. def post_json(url, data, headers={}):
  466. postdata = json.dumps(data)
  467. headers['Content-Type'] = 'application/json'
  468. req = urllib2.Request(url, postdata, headers)
  469. req.add_header('User-Agent', UA)
  470. response = urllib2.urlopen(req)
  471. data = response.read()
  472. response.close()
  473. return data
  474. #def run_parallel_in_threads(target, args_list):
  475. #result = Queue.Queue()
  476. ## wrapper to collect return value in a Queue
  477. #def task_wrapper(*args):
  478. #result.put(target(*args))
  479. #threads = [threading.Thread(target=task_wrapper, args=args) for args in args_list]
  480. #for t in threads:
  481. #t.start()
  482. #for t in threads:
  483. #t.join()
  484. #return result
  485. def substr(data, start, end):
  486. i1 = data.find(start)
  487. i2 = data.find(end, i1)
  488. return data[i1:i2]
  489. def save_to_file(url, file):
  490. try:
  491. return save_data_to_file(request(url), file)
  492. except:
  493. traceback.print_exc()
  494. def save_data_to_file(data, file):
  495. try:
  496. f = open(file, 'wb')
  497. f.write(data)
  498. f.close()
  499. info('File %s saved' % file)
  500. return True
  501. except:
  502. traceback.print_exc()
  503. def read_file(file):
  504. if not os.path.exists(file):
  505. return ''
  506. f = open(file, 'r')
  507. data = f.read()
  508. f.close()
  509. return data
  510. def _substitute_entity(match):
  511. ent = match.group(3)
  512. if match.group(1) == '#':
  513. # decoding by number
  514. if match.group(2) == '':
  515. # number is in decimal
  516. return unichr(int(ent))
  517. elif match.group(2) == 'x':
  518. # number is in hex
  519. return unichr(int('0x' + ent, 16))
  520. else:
  521. # they were using a name
  522. cp = n2cp.get(ent)
  523. if cp:
  524. return unichr(cp)
  525. else:
  526. return match.group()
  527. def decode_html(data):
  528. if not type(data) == str:
  529. return data
  530. try:
  531. if not type(data) == unicode:
  532. data = unicode(data, 'utf-8', errors='ignore')
  533. entity_re = re.compile(r'&(#?)(x?)(\w+);')
  534. return entity_re.subn(_substitute_entity, data)[0]
  535. except:
  536. traceback.print_exc()
  537. print[data]
  538. return data
  539. def unescape(s0):
  540. #s2 = re.sub("&#\w+;",HTMLParser.HTMLParser().unescape("\1"),s)
  541. s0 = s0.replace("&amp;","&").replace("&quot;", '"')
  542. for s in re.findall("&#\w+;",s0):
  543. s2 = HTMLParser.HTMLParser().unescape(s)
  544. if isinstance(s0,str):
  545. s2 = s2.encode("utf8")
  546. s0 = s0.replace(s,s2)
  547. pass
  548. return s0
  549. def debug(text):
  550. if LOG > 1:
  551. print('[DEBUG] ' + str([text]))
  552. def info(text):
  553. if LOG > 0:
  554. print('[INFO] ' + str([text]))
  555. def error(text):
  556. print('[ERROR] ' + str([text]))
  557. _diacritic_replace = {u'\u00f3': 'o',
  558. u'\u0213': '-',
  559. u'\u00e1': 'a',
  560. u'\u010d': 'c',
  561. u'\u010c': 'C',
  562. u'\u010f': 'd',
  563. u'\u010e': 'D',
  564. u'\u00e9': 'e',
  565. u'\u011b': 'e',
  566. u'\u00ed': 'i',
  567. u'\u0148': 'n',
  568. u'\u0159': 'r',
  569. u'\u0161': 's',
  570. u'\u0165': 't',
  571. u'\u016f': 'u',
  572. u'\u00fd': 'y',
  573. u'\u017e': 'z',
  574. u'\xed': 'i',
  575. u'\xe9': 'e',
  576. u'\xe1': 'a',
  577. }
  578. def replace_diacritic(string):
  579. ret = []
  580. for char in string:
  581. if char in _diacritic_replace:
  582. ret.append(_diacritic_replace[char])
  583. else:
  584. ret.append(char)
  585. return ''.join(ret)
  586. def params(url=None):
  587. if not url:
  588. url = sys.argv[2]
  589. param = {}
  590. paramstring = url
  591. if len(paramstring) >= 2:
  592. params = url
  593. cleanedparams = params.replace('?', '')
  594. if (params[len(params) - 1] == '/'):
  595. params = params[0:len(params) - 2]
  596. pairsofparams = cleanedparams.split('&')
  597. param = {}
  598. for i in range(len(pairsofparams)):
  599. splitparams = {}
  600. splitparams = pairsofparams[i].split('=')
  601. if (len(splitparams)) == 2:
  602. param[splitparams[0]] = splitparams[1]
  603. for p in param.keys():
  604. param[p] = param[p].decode('hex')
  605. return param
  606. def int_to_base(number, base):
  607. digs = string.digits + string.letters
  608. if number < 0:
  609. sign = -1
  610. elif number == 0:
  611. return digs[0]
  612. else:
  613. sign = 1
  614. number *= sign
  615. digits = []
  616. while number:
  617. digits.append(digs[number % base])
  618. number /= base
  619. if sign < 0:
  620. digits.append('-')
  621. digits.reverse()
  622. return ''.join(digits)
  623. def extract_jwplayer_setup(data):
  624. """
  625. Extracts jwplayer setup configuration and returns it as a dictionary.
  626. :param data: A string to extract the setup from
  627. :return: A dictionary containing the setup configuration
  628. """
  629. data = re.search(r'<script.+?}\(\'(.+)\',\d+,\d+,\'([\w\|]+)\'.*</script>', data, re.I | re.S)
  630. if data:
  631. replacements = data.group(2).split('|')
  632. data = data.group(1)
  633. for i in reversed(range(len(replacements))):
  634. if len(replacements[i]) > 0:
  635. data = re.sub(r'\b%s\b' % int_to_base(i, 36), replacements[i], data)
  636. data = re.search(r'\.setup\(([^\)]+?)\);', data)
  637. if data:
  638. return json.loads(data.group(1).decode('string_escape'))
  639. #return demjson.decode(data.group(1).decode('string_escape')) ### III
  640. return None
  641. def datetime_from_utc_to_local(utc_datetime):
  642. now_timestamp = time.time()
  643. offset = datetime.datetime.fromtimestamp(now_timestamp) - datetime.datetime.utcfromtimestamp(now_timestamp)
  644. return utc_datetime + offset
  645. def datetime_from_zulu_to_utc(data):
  646. d = map(int, data.split("T")[0].split("-"))
  647. h = map(int, data.split("T")[1][:-1].split(":"))
  648. dt = datetime.datetime(d[0], d[1], d[2], h[0], h[1], h[2])
  649. return dt
  650. #def parse_html(url):
  651. # return BeautifulSoup(request(url), 'html5lib', from_encoding='utf-8')
  652. if __name__ == "__main__":
  653. datetime_from_zulu_to_utc('2018-06-24T06:10:00Z')
  654. s = open("The LEGO Ninjago Movie (2017).lv.vtt").read()
  655. #s2 = vtt2srt(s)
  656. #open("The LEGO Ninjago Movie (2017).lv.srt", "w").write(s2)
  657. #s = 'B\xc4\x93thovena D\xc4\x81rgumu Taka (2014)/Beethoven&#x27;s Treasure [LV]'
  658. #s = s.decode("utf8")
  659. #s=unescape(s)
  660. #url = "http://localhost:88/https://walterebert.com/playground/video/hls/ts/480x270.m3u8?token=xxxx~User-Agent=Enigma2~Cookie=xxxxx"
  661. #url = "http://hyt4d6.vkcache.com/secip/0/UMQ3q2gNjTlOPnEVm3iTiA/ODAuMjMyLjI0MC42/1479610800/hls-vod-s3/flv/api/files/videos/2015/09/11/144197748923a22.mp4.m3u8http://hyt4d6.vkcache.com/secip/0/Y-ZA1qRm8toplc0dN_L6_w/ODAuMjMyLjI0MC42/1479654000/hls-vod-s3/flv/api/files/videos/2015/09/11/144197748923a22.mp4.m3u8"
  662. #headers = {"User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 9_2 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/47.0.2526.70 Mobile/13C71 Safari/601.1.46"}
  663. #url = "http://str1e.lattelecom.tv/mobile-vod/mp4:sf_fantastic_beasts_and_where_to_find_them_en_hd.mp4/playlist.m3u8?resource_id=fantastic_beasts_and_where_to_find_them&auth_token=6NAvMFDG+rYTAc4hb5JeL2bmsaRR7bAE23M6KDmhKYOGyXoo0gDpJUE9scYy+nQmfbgk03cWMe9MuXWSH1GqwolEk2jOQ/8Mrg7tOdbwrA8zM7nmkfCZPqQkwajZN4mfSJQVKHqXqJ8="
  664. #headers={}
  665. #print url
  666. #url = "replay::tiesraide/ltv1/"
  667. #url = "ltc::content/live-streams/103?include=quality"
  668. #urlp = streamproxy_encode2(url,headers)
  669. #print urlp
  670. #url2,headers2 = streamproxy_decode2(urlp)
  671. #print url2
  672. #player(urlp)
  673. pass