Python module (submodule repositary), which provides content (video streams) from various online stream sources to corresponding Enigma2, Kodi, Plex plugins

util.py 24KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765
  1. # -*- coding: UTF-8 -*-
  2. # /*
  3. # * Copyright (C) 2011 Libor Zoubek,ivars777
  4. # *
  5. # *
  6. # * This Program is free software; you can redistribute it and/or modify
  7. # * it under the terms of the GNU General Public License as published by
  8. # * the Free Software Foundation; either version 2, or (at your option)
  9. # * any later version.
  10. # *
  11. # * This Program is distributed in the hope that it will be useful,
  12. # * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # * GNU General Public License for more details.
  15. # *
  16. # * You should have received a copy of the GNU General Public License
  17. # * along with this program; see the file COPYING. If not, write to
  18. # * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  19. # * http://www.gnu.org/copyleft/gpl.html
  20. # *
  21. # */
  22. import os, sys, re
  23. import urllib, urllib2
  24. import datetime, time
  25. import traceback
  26. import cookielib
  27. import requests
  28. try:
  29. from requests.packages.urllib3.exceptions import InsecureRequestWarning
  30. requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
  31. except:
  32. pass
  33. from htmlentitydefs import name2codepoint as n2cp
  34. import HTMLParser
  35. import StringIO
  36. #import threading
  37. #import Queue
  38. import pickle
  39. import string
  40. import simplejson as json
  41. #from demjson import demjson
  42. #import demjson
  43. import json
  44. #from bs4 import BeautifulSoup
  45. UA = 'Mozilla/6.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.9.0.5) Gecko/2008092417 Firefox/3.0.3'
  46. LOG = 2
  47. _cookie_jar = None
  48. CACHE_COOKIES = 'cookies'
  49. def system():
  50. if "kodi" in sys.executable.lower():
  51. return "kodi"
  52. elif sys.platform == "win32":
  53. return "windows"
  54. elif sys.platform == "linux2":
  55. return "enigma2"
  56. else:
  57. return "unknown"
  58. def nfo2xml(nfo_dict):
  59. nfo_type,nfo = next(nfo_dict.iteritems())
  60. s= "<%s>\n"%nfo_type.encode("utf8")
  61. for k,v in nfo.iteritems():
  62. if isinstance(v,list):
  63. for v2 in v:
  64. if isinstance(v2,unicode): v2 = v2.encode("utf8")
  65. s += " <%s>%s</%s>\n"%(k.encode("utf8"), v2, k.encode("utf8"))
  66. else:
  67. if isinstance(v,unicode): v = v.encode("utf8")
  68. s += " <%s>%s</%s>\n"%(k.encode("utf8"), v, k.encode("utf8"))
  69. s += "</%s>\n"%nfo_type.encode("utf8")
  70. return s
  71. def nfo2desc(nfo):
  72. if not "title" in nfo:
  73. nfo_type, nfo = next(nfo.iteritems())
  74. desc = nfo2title(nfo)
  75. dd = lambda t: "\n" + nfo[t] if t in nfo and nfo[t] else ""
  76. dd2 = lambda t: "\n" + ",".join(nfo[t]) if t in nfo and nfo[t] else ""
  77. def ddd(t,title=""):
  78. if title:
  79. title = title + ": "
  80. if t in nfo and nfo[t]:
  81. if isinstance(nfo[t],list):
  82. d = "\n" + title + ",".join(nfo[t])
  83. else:
  84. d = "\n" + title + nfo[t]
  85. else:
  86. d = ""
  87. return d.encode("utf8") if isinstance(d, unicode) else d
  88. desc += ddd("genre","Genre")
  89. desc += ddd("runtime","Length")
  90. desc += ddd("director","Director")
  91. desc += ddd("actor","Actors")
  92. desc += ddd("language","Languages")
  93. desc += ddd("quality","Quality")
  94. desc += ddd("tagline")
  95. if "plot" in nfo and "tagline" in nfo and nfo["tagline"] <> nfo["plot"]:
  96. desc += ddd("plot")
  97. elif "plot" in nfo and not "tagline" in nfo:
  98. desc += ddd("plot")
  99. return desc.encode("utf8") if isinstance(desc,unicode) else desc
  100. def nfo2title(nfo):
  101. if not "title" in nfo:
  102. nfo_type, nfo = next(nfo.iteritems())
  103. title = nfo["title"]
  104. if "originaltitle" in nfo and nfo["originaltitle"] and nfo["originaltitle"]<>nfo["title"]:
  105. title +=" ~ "+nfo["originaltitle"]
  106. se = ""
  107. if "season" in nfo and nfo["season"]:
  108. se += "S%s" % nfo["season"]
  109. if "episode" in nfo and nfo["episode"]:
  110. se += "E%s" % nfo["episode"]
  111. if se:
  112. title += " (%s)" % se
  113. elif "year" in nfo and nfo["year"]:
  114. title += " (%s)"%nfo["year"]
  115. return title.encode("utf8") if isinstance(title,unicode) else title
  116. def stream_type(data):
  117. data = data.lower()
  118. m = re.search(r"^(\w+)://", data)
  119. prefix = m.group(1) if m else ""
  120. if prefix in ("http","https"):
  121. if ".m3u8" in data:
  122. return "hls"
  123. elif ".mpd" in data:
  124. return "dash"
  125. else:
  126. return "http"
  127. else:
  128. return prefix
  129. def check_version(package,url="http://feed.blue.lv/Packages"):
  130. "Return current package version from OPKG feed"
  131. url = "http://feed.blue.lv/Packages"
  132. r = requests.get(url)
  133. if not r.ok:
  134. return ""
  135. m = re.search("Package: %s\nVersion: (.+?)\n"%package, r.content)
  136. if not m:
  137. return ""
  138. return m.group(1)
  139. SPLIT_CHAR = "~"
  140. SPLIT_CODE = urllib.quote(SPLIT_CHAR)
  141. EQ_CODE = urllib.quote("=")
  142. COL_CODE = urllib.quote(":")
  143. SPACE_CODE = urllib.quote(" ")
  144. def make_fname(title):
  145. "Make file name from title"
  146. title = title.strip()
  147. fname0 = re.sub("[/\n\r\t,:]"," ",title)
  148. fname0 = re.sub("['""]","",fname0)
  149. return fname0
  150. def hls_base(url):
  151. url2 = url.split("?")[0]
  152. url2 = "/".join(url2.split("/")[0:-1])+ "/"
  153. return url2
  154. def stream_change(stream):
  155. #return stream # TODO
  156. if "resolver" in stream and stream["resolver"] in ("viaplay","hqq","filmas") or \
  157. "surl" in stream and re.search("https*://(hqq|goo\.gl)",stream["surl"]):
  158. stream["url"] = streamproxy_encode(stream["url"],stream["headers"])
  159. stream["headers"] = {}
  160. return stream
  161. else:
  162. return stream
  163. def streamproxy_encode(url,headers=[],proxy_url=None):
  164. PROXY_URL = "http://localhost:8880/"
  165. if not "?" in url:
  166. url = url+"?"
  167. url2 = url.replace(SPLIT_CHAR,SPLIT_CODE).replace(":",COL_CODE).replace(" ",SPACE_CODE)
  168. if not proxy_url:
  169. proxy_url = PROXY_URL
  170. url2 = proxy_url + url2
  171. if headers:
  172. headers2 = []
  173. for h in headers:
  174. headers2.append("%s=%s"%(h,headers[h].replace("=",EQ_CODE).replace(SPLIT_CHAR,SPLIT_CODE).replace(" ",SPACE_CODE)))
  175. headers2 = SPLIT_CHAR.join(headers2)
  176. url2 = url2+SPLIT_CHAR+headers2
  177. #return url2.encode("utf8") if isinstance(url2,unicode) else url2
  178. return url2
  179. def streamproxy_decode(urlp):
  180. import urlparse
  181. path = urlp.replace(re.search("http://[^/]+",urlp).group(0),"")
  182. p = path.split(SPLIT_CHAR)
  183. url = urllib.unquote(p[0][1:])
  184. #headers = {"User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 9_2 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/47.0.2526.70 Mobile/13C71 Safari/601.1.46"}
  185. headers={}
  186. if len(p)>1:
  187. for h in p[1:]:
  188. #h = urllib.unquote()
  189. headers[h.split("=")[0]]=urllib.unquote(h.split("=")[1])
  190. return url,headers
  191. def streamproxy_encode2(url,headers=[],proxy_url=None):
  192. PROXY_URL = "http://localhost:8880/"
  193. #url2 = url.replace(SPLIT_CHAR,SPLIT_CODE).replace(":",COL_CODE).replace(" ",SPACE_CODE)
  194. url2 = urllib.quote_plus(url)
  195. if not proxy_url:
  196. proxy_url = PROXY_URL
  197. url2 = proxy_url + url2+"/?"
  198. if headers:
  199. headers2 = []
  200. for h in headers:
  201. headers2.append("%s=%s"%(h,headers[h].replace("=",EQ_CODE).replace(SPLIT_CHAR,SPLIT_CODE).replace(" ",SPACE_CODE)))
  202. headers2 = SPLIT_CHAR.join(headers2)
  203. url2 = url2+SPLIT_CHAR+headers2
  204. return url2
  205. def streamproxy_decode2(urlp):
  206. path = urlp.replace(re.search("http://[^/]+",urlp).group(0),"")
  207. p = path.split(SPLIT_CHAR)
  208. url = urllib.unquote_plus(p[0][1:-2])
  209. #headers = {"User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 9_2 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/47.0.2526.70 Mobile/13C71 Safari/601.1.46"}
  210. headers={}
  211. if len(p)>1:
  212. for h in p[1:]:
  213. #h = urllib.unquote()
  214. headers[h.split("=")[0]]=urllib.unquote(h.split("=")[1])
  215. return url,headers
  216. def streamproxy_encode3(cmd, data, proxy_url=None, headers=None, qs=None):
  217. PROXY_URL = "http://localhost:8880/"
  218. #url2 = url.replace(SPLIT_CHAR,SPLIT_CODE).replace(":",COL_CODE).replace(" ",SPACE_CODE)
  219. data2 = urllib.quote_plus(data)
  220. if not proxy_url:
  221. proxy_url = PROXY_URL
  222. urlp = proxy_url + cmd + "/" + data2 + "/"
  223. qs2 = []
  224. if headers:
  225. headers2 = []
  226. for h in headers:
  227. headers2.append("%s=%s"%(h,urllib.quote_plus(headers[h])))
  228. headers2 = SPLIT_CHAR.join(headers2)
  229. qs2.append("headers="+urllib.quote_plus(headers2))
  230. if qs:
  231. for qv in qs:
  232. qs2.append(qv+"="+urllib.quote_plus(qs[qv]))
  233. if qs2:
  234. qs2 = "&".join(qs2)
  235. if qs2:
  236. urlp = urlp + "?" + qs2
  237. return urlp
  238. def streamproxy_decode3(urlp):
  239. m = re.search("http://[^/]+",urlp)
  240. path = urlp.replace(m.group(0),"") if m else urlp
  241. cmd = path.split("/")[1]
  242. data = path.split("/")[2].split("?")[0]
  243. data = urllib.unquote_plus(data)
  244. qs = path.split("?")[1] if "?" in path else ""
  245. qs2 = {}
  246. headers = {}
  247. if qs:
  248. qs = qs.split("&")
  249. for q in qs:
  250. qk = q.split("=")[0]
  251. if qk == "headers":
  252. hh = urllib.unquote_plus(q.split("=")[1])
  253. hh = hh.split(SPLIT_CHAR)
  254. for h in hh:
  255. headers[h.split("=")[0]] = urllib.unquote_plus(h.split("=")[1])
  256. else:
  257. qv = urllib.unquote_plus(q.split("=")[1]) if "=" in q else ""
  258. qs2[qk] = qv
  259. return cmd, data, headers, qs2
  260. class Captions(object):
  261. def __init__(self,uri):
  262. self.uri = uri
  263. self.subs = []
  264. self.styles = {}
  265. if uri.startswith("http"):
  266. r = requests.get(uri)
  267. if r.status_code == 200:
  268. self.loads(r.content)
  269. def loads(self,s):
  270. if "WEBVTT" in s[:s.find("\n")]: # vtt captions
  271. self.load_vtt(s)
  272. elif "<?xml" in s[:s.find("\n")]:
  273. self.load_ttaf(s)
  274. else:
  275. self.load_vtt(s) # TODO
  276. def load_ttaf(self,s):
  277. for r2 in re.findall("<style .+?/>", s):
  278. st = {}
  279. for a in re.findall(r'(\w+)="([^ "]+)"', r2):
  280. st[a[0]] = a[1]
  281. if a[0] == "id":
  282. sid = a[1]
  283. self.styles[sid] = st
  284. for r2 in re.findall("<p .+?</p>", s):
  285. sub = {}
  286. sub["begin"] = str2sec(re.search('begin="([^"]+)"', r2).group(1)) if re.search('begin="([^"]+)"', r2) else -1
  287. sub["end"] = str2sec(re.search('end="([^"]+)"', r2).group(1)) if re.search('end="([^"]+)"', r2) else -1
  288. sub["style"] = re.search('style="([^"]+)"', r2).group(1) if re.search('style="([^"]+)"', r2) else None
  289. sub["text"] = re.search("<p[^>]+>(.+)</p>", r2).group(1).replace("\n","")
  290. sub["text"] = re.sub("<br\s*?/>","\n",sub["text"])
  291. sub["text"] = re.sub("<.+?>"," ",sub["text"])
  292. self.subs.append(sub)
  293. pass
  294. def load_vtt(self,s):
  295. f = StringIO.StringIO(s)
  296. while True:
  297. line = f.readline()
  298. if not line:
  299. break
  300. m = re.search(r"([\d\.\,:]+)\s*-->\s*([\d\.\,\:]+)",line)
  301. if m:
  302. sub = {}
  303. sub["begin"] = str2sec(m.group(1))
  304. sub["end"] = str2sec(m.group(2))
  305. sub["style"] = None
  306. sub["text"] = []
  307. line = f.readline()
  308. while line.strip():
  309. txt = line.strip()
  310. if isinstance(txt,unicode):
  311. txt = txt.encode("utf8")
  312. sub["text"].append(txt)
  313. line = f.readline()
  314. sub["text"] = "\n".join(sub["text"])
  315. self.subs.append(sub)
  316. else:
  317. continue
  318. pass
  319. def get_srt(self):
  320. out = ""
  321. i = 0
  322. for sub in self.subs:
  323. i +=1
  324. begin = sub["begin"]
  325. begin = "%s,%03i"%(str(datetime.timedelta(seconds=begin/1000)),begin%1000)
  326. end = sub["end"]
  327. end = "%s,%03i"%(str(datetime.timedelta(seconds=end/1000)),end%1000)
  328. txt2 = sub["text"]
  329. out += "%s\n%s --> %s\n%s\n\n\n"%(i,begin,end,txt2)
  330. return out
  331. def str2sec(r):
  332. # Convert str time to miliseconds
  333. r= r.replace(",",".")
  334. m = re.search(r"(\d+\:)*(\d+)\:(\d+\.\d+)", r)
  335. if m:
  336. sec = int(m.group(1)[:-1])*60*60*1000 if m.group(1) else 0
  337. sec += int(m.group(2))*60*1000 + int(float(m.group(3))*1000)
  338. return sec
  339. else:
  340. return -1
  341. #c = Captions("http://195.13.216.2/mobile-vod/mp4:lb_barbecue_fr_lq.mp4/lb_barbecue_lv.vtt")
  342. #c = Captions("http://www.bbc.co.uk/iplayer/subtitles/ng/modav/bUnknown-0edd6227-0f38-411c-8d46-fa033c4c61c1_b05ql1s3_1479853893356.xml")
  343. #url = "http://195.13.216.2/mobile-vod/mp4:ac_now_you_see_me_2_en_lq.mp4/ac_now_you_see_me_2_lv.vtt"
  344. #c = Captions(url)
  345. #pass
  346. def ttaf2srt(s):
  347. out = u""
  348. i = 0
  349. for p,txt in re.findall("<p ([^>]+)>(.+?)</p>", s, re.DOTALL):
  350. i +=1
  351. begin = re.search('begin="(.+?)"',p).group(1)
  352. begin = begin.replace(".",",")
  353. end = re.search('end="(.+?)"',p).group(1)
  354. end = end.replace(".",",")
  355. txt2 = re.sub("<br */>","\n",txt)
  356. out += "%s\n%s --> %s\n%s\n\n"%(i,begin,end,txt2)
  357. return out
  358. def vtt2srt(s):
  359. out = ""
  360. i = 0
  361. # result = re.findall(r"([\d:\. ]+-->[\d:\. ]+)\n(.+?\n)+", r.content)
  362. for t1, t2, txt in re.findall(r"([\d:\.]+)\s*-->\s*([\d:\.]+)\n(.+?\n)+", s):
  363. i +=1
  364. if len(t1.split(":")) == 2:
  365. t1 = "00:" + t1
  366. t1 = t1.replace(".", ",")
  367. if len(t2.split(":")) == 2:
  368. t2 = "00:" + t2
  369. t2 = t2.replace(".", ",")
  370. out += "%s\n%s --> %s\n%s\n\n"%(i,t1,t2,txt)
  371. return out
  372. def item():
  373. """Default item content"""
  374. stream0 = {
  375. 'name': '', #
  376. 'url': '',
  377. 'quality': '?',
  378. 'surl': '',
  379. 'subs': [],
  380. 'headers': {},
  381. "desc":"",
  382. "img":"",
  383. "lang":"",
  384. "type":"",
  385. "resolver":"",
  386. "order":0,
  387. "live":False
  388. }
  389. return stream0
  390. class _StringCookieJar(cookielib.LWPCookieJar):
  391. def __init__(self, string=None, filename=None, delayload=False, policy=None):
  392. cookielib.LWPCookieJar.__init__(self, filename, delayload, policy)
  393. if string and len(string) > 0:
  394. self._cookies = pickle.loads(str(string))
  395. def dump(self):
  396. return pickle.dumps(self._cookies)
  397. def init_urllib(cache=None):
  398. """
  399. Initializes urllib cookie handler
  400. """
  401. global _cookie_jar
  402. data = None
  403. if cache is not None:
  404. data = cache.get(CACHE_COOKIES)
  405. _cookie_jar = _StringCookieJar(data)
  406. opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(_cookie_jar))
  407. urllib2.install_opener(opener)
  408. def cache_cookies(cache):
  409. """
  410. Saves cookies to cache
  411. """
  412. global _cookie_jar
  413. if _cookie_jar:
  414. cache.set(CACHE_COOKIES, _cookie_jar.dump())
  415. def request0(url, headers={}):
  416. debug('request: %s' % url)
  417. req = urllib2.Request(url, headers=headers)
  418. req.add_header('User-Agent', UA)
  419. try:
  420. response = urllib2.urlopen(req)
  421. data = response.read()
  422. response.close()
  423. except urllib2.HTTPError, error:
  424. data = error.read()
  425. debug('len(data) %s' % len(data))
  426. return data
  427. def request(url, headers={}):
  428. debug('request: %s' % url)
  429. #req = urllib2.Request(url, headers=headers)
  430. #req.add_header('User-Agent', UA)
  431. if 'User-Agent' not in headers:
  432. headers['User-Agent']= UA
  433. try:
  434. r = requests.get(url, headers=headers)
  435. data = r.content
  436. except:
  437. data = r.content
  438. debug('len(data) %s' % len(data))
  439. return data
  440. def post(url, data, headers={}):
  441. postdata = urllib.urlencode(data)
  442. #req = urllib2.Request(url, postdata, headers)
  443. #req.add_header('User-Agent', UA)
  444. import requests
  445. if 'User-Agent' not in headers:
  446. headers['User-Agent']= UA
  447. try:
  448. r = requests.post(url, data=postdata,headers=headers)
  449. data = r.content
  450. except urllib2.HTTPError, error:
  451. data = r.content
  452. return data
  453. def post0(url, data, headers={}):
  454. postdata = urllib.urlencode(data)
  455. req = urllib2.Request(url, postdata, headers)
  456. req.add_header('User-Agent', UA)
  457. try:
  458. response = urllib2.urlopen(req)
  459. data = response.read()
  460. response.close()
  461. except urllib2.HTTPError, error:
  462. data = error.read()
  463. return data
  464. def post_json(url, data, headers={}):
  465. postdata = json.dumps(data)
  466. headers['Content-Type'] = 'application/json'
  467. req = urllib2.Request(url, postdata, headers)
  468. req.add_header('User-Agent', UA)
  469. response = urllib2.urlopen(req)
  470. data = response.read()
  471. response.close()
  472. return data
  473. #def run_parallel_in_threads(target, args_list):
  474. #result = Queue.Queue()
  475. ## wrapper to collect return value in a Queue
  476. #def task_wrapper(*args):
  477. #result.put(target(*args))
  478. #threads = [threading.Thread(target=task_wrapper, args=args) for args in args_list]
  479. #for t in threads:
  480. #t.start()
  481. #for t in threads:
  482. #t.join()
  483. #return result
  484. def substr(data, start, end):
  485. i1 = data.find(start)
  486. i2 = data.find(end, i1)
  487. return data[i1:i2]
  488. def save_to_file(url, file):
  489. try:
  490. return save_data_to_file(request(url), file)
  491. except:
  492. traceback.print_exc()
  493. def save_data_to_file(data, file):
  494. try:
  495. f = open(file, 'wb')
  496. f.write(data)
  497. f.close()
  498. info('File %s saved' % file)
  499. return True
  500. except:
  501. traceback.print_exc()
  502. def read_file(file):
  503. if not os.path.exists(file):
  504. return ''
  505. f = open(file, 'r')
  506. data = f.read()
  507. f.close()
  508. return data
  509. def _substitute_entity(match):
  510. ent = match.group(3)
  511. if match.group(1) == '#':
  512. # decoding by number
  513. if match.group(2) == '':
  514. # number is in decimal
  515. return unichr(int(ent))
  516. elif match.group(2) == 'x':
  517. # number is in hex
  518. return unichr(int('0x' + ent, 16))
  519. else:
  520. # they were using a name
  521. cp = n2cp.get(ent)
  522. if cp:
  523. return unichr(cp)
  524. else:
  525. return match.group()
  526. def decode_html(data):
  527. if not type(data) == str:
  528. return data
  529. try:
  530. if not type(data) == unicode:
  531. data = unicode(data, 'utf-8', errors='ignore')
  532. entity_re = re.compile(r'&(#?)(x?)(\w+);')
  533. return entity_re.subn(_substitute_entity, data)[0]
  534. except:
  535. traceback.print_exc()
  536. print[data]
  537. return data
  538. def unescape(s0):
  539. #s2 = re.sub("&#\w+;",HTMLParser.HTMLParser().unescape("\1"),s)
  540. s0 = s0.replace("&amp;","&").replace("&quot;", '"')
  541. for s in re.findall("&#\w+;",s0):
  542. s2 = HTMLParser.HTMLParser().unescape(s)
  543. if isinstance(s0,str):
  544. s2 = s2.encode("utf8")
  545. s0 = s0.replace(s,s2)
  546. pass
  547. return s0
  548. def debug(text):
  549. if LOG > 1:
  550. print('[DEBUG] ' + str([text]))
  551. def info(text):
  552. if LOG > 0:
  553. print('[INFO] ' + str([text]))
  554. def error(text):
  555. print('[ERROR] ' + str([text]))
  556. _diacritic_replace = {u'\u00f3': 'o',
  557. u'\u0213': '-',
  558. u'\u00e1': 'a',
  559. u'\u010d': 'c',
  560. u'\u010c': 'C',
  561. u'\u010f': 'd',
  562. u'\u010e': 'D',
  563. u'\u00e9': 'e',
  564. u'\u011b': 'e',
  565. u'\u00ed': 'i',
  566. u'\u0148': 'n',
  567. u'\u0159': 'r',
  568. u'\u0161': 's',
  569. u'\u0165': 't',
  570. u'\u016f': 'u',
  571. u'\u00fd': 'y',
  572. u'\u017e': 'z',
  573. u'\xed': 'i',
  574. u'\xe9': 'e',
  575. u'\xe1': 'a',
  576. }
  577. def replace_diacritic(string):
  578. ret = []
  579. for char in string:
  580. if char in _diacritic_replace:
  581. ret.append(_diacritic_replace[char])
  582. else:
  583. ret.append(char)
  584. return ''.join(ret)
  585. def params(url=None):
  586. if not url:
  587. url = sys.argv[2]
  588. param = {}
  589. paramstring = url
  590. if len(paramstring) >= 2:
  591. params = url
  592. cleanedparams = params.replace('?', '')
  593. if (params[len(params) - 1] == '/'):
  594. params = params[0:len(params) - 2]
  595. pairsofparams = cleanedparams.split('&')
  596. param = {}
  597. for i in range(len(pairsofparams)):
  598. splitparams = {}
  599. splitparams = pairsofparams[i].split('=')
  600. if (len(splitparams)) == 2:
  601. param[splitparams[0]] = splitparams[1]
  602. for p in param.keys():
  603. param[p] = param[p].decode('hex')
  604. return param
  605. def int_to_base(number, base):
  606. digs = string.digits + string.letters
  607. if number < 0:
  608. sign = -1
  609. elif number == 0:
  610. return digs[0]
  611. else:
  612. sign = 1
  613. number *= sign
  614. digits = []
  615. while number:
  616. digits.append(digs[number % base])
  617. number /= base
  618. if sign < 0:
  619. digits.append('-')
  620. digits.reverse()
  621. return ''.join(digits)
  622. def extract_jwplayer_setup(data):
  623. """
  624. Extracts jwplayer setup configuration and returns it as a dictionary.
  625. :param data: A string to extract the setup from
  626. :return: A dictionary containing the setup configuration
  627. """
  628. data = re.search(r'<script.+?}\(\'(.+)\',\d+,\d+,\'([\w\|]+)\'.*</script>', data, re.I | re.S)
  629. if data:
  630. replacements = data.group(2).split('|')
  631. data = data.group(1)
  632. for i in reversed(range(len(replacements))):
  633. if len(replacements[i]) > 0:
  634. data = re.sub(r'\b%s\b' % int_to_base(i, 36), replacements[i], data)
  635. data = re.search(r'\.setup\(([^\)]+?)\);', data)
  636. if data:
  637. return json.loads(data.group(1).decode('string_escape'))
  638. #return demjson.decode(data.group(1).decode('string_escape')) ### III
  639. return None
  640. def datetime_from_utc_to_local(utc_datetime):
  641. now_timestamp = time.time()
  642. offset = datetime.datetime.fromtimestamp(now_timestamp) - datetime.datetime.utcfromtimestamp(now_timestamp)
  643. return utc_datetime + offset
  644. def datetime_from_zulu_to_utc(data):
  645. d = map(int, data.split("T")[0].split("-"))
  646. h = map(int, data.split("T")[1][:-1].split(":"))
  647. dt = datetime.datetime(d[0], d[1], d[2], h[0], h[1], h[2])
  648. return dt
  649. #def parse_html(url):
  650. # return BeautifulSoup(request(url), 'html5lib', from_encoding='utf-8')
  651. if __name__ == "__main__":
  652. datetime_from_zulu_to_utc('2018-06-24T06:10:00Z')
  653. s = open("The LEGO Ninjago Movie (2017).lv.vtt").read()
  654. #s2 = vtt2srt(s)
  655. #open("The LEGO Ninjago Movie (2017).lv.srt", "w").write(s2)
  656. #s = 'B\xc4\x93thovena D\xc4\x81rgumu Taka (2014)/Beethoven&#x27;s Treasure [LV]'
  657. #s = s.decode("utf8")
  658. #s=unescape(s)
  659. #url = "http://localhost:88/https://walterebert.com/playground/video/hls/ts/480x270.m3u8?token=xxxx~User-Agent=Enigma2~Cookie=xxxxx"
  660. #url = "http://hyt4d6.vkcache.com/secip/0/UMQ3q2gNjTlOPnEVm3iTiA/ODAuMjMyLjI0MC42/1479610800/hls-vod-s3/flv/api/files/videos/2015/09/11/144197748923a22.mp4.m3u8http://hyt4d6.vkcache.com/secip/0/Y-ZA1qRm8toplc0dN_L6_w/ODAuMjMyLjI0MC42/1479654000/hls-vod-s3/flv/api/files/videos/2015/09/11/144197748923a22.mp4.m3u8"
  661. #headers = {"User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 9_2 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/47.0.2526.70 Mobile/13C71 Safari/601.1.46"}
  662. #url = "http://str1e.lattelecom.tv/mobile-vod/mp4:sf_fantastic_beasts_and_where_to_find_them_en_hd.mp4/playlist.m3u8?resource_id=fantastic_beasts_and_where_to_find_them&auth_token=6NAvMFDG+rYTAc4hb5JeL2bmsaRR7bAE23M6KDmhKYOGyXoo0gDpJUE9scYy+nQmfbgk03cWMe9MuXWSH1GqwolEk2jOQ/8Mrg7tOdbwrA8zM7nmkfCZPqQkwajZN4mfSJQVKHqXqJ8="
  663. #headers={}
  664. #print url
  665. #url = "replay::tiesraide/ltv1/"
  666. #url = "ltc::content/live-streams/103?include=quality"
  667. #urlp = streamproxy_encode2(url,headers)
  668. #print urlp
  669. #url2,headers2 = streamproxy_decode2(urlp)
  670. #print url2
  671. #player(urlp)
  672. pass