Python module (submodule repositary), which provides content (video streams) from various online stream sources to corresponding Enigma2, Kodi, Plex plugins

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293
  1. #from boxbranding import getMachineBrand, getMachineName
  2. import sys,os, os.path, re
  3. import urlparse, requests
  4. from twisted.web import client
  5. from twisted.internet import reactor, defer, ssl
  6. USER_AGENT = "Enigma2 HbbTV/1.1.1 (+PVR+RTSP+DL;OpenATV;;;)"
  7. #####################################################################################################
  8. class HTTPProgressDownloader(client.HTTPDownloader):
  9. def __init__(self, url, outfile, headers=None):
  10. agent = USER_AGENT
  11. if headers and "user-agent" in headers:
  12. agent = headers["user-agent"]
  13. if headers and "User-Agent" in headers:
  14. agent = headers["User-Agent"]
  15. client.HTTPDownloader.__init__(self, url, outfile, headers=headers, agent=agent)
  16. self.status = None
  17. self.progress_callback = None
  18. self.deferred = defer.Deferred()
  19. def noPage(self, reason):
  20. if self.status == "304":
  21. print reason.getErrorMessage()
  22. client.HTTPDownloader.page(self, "")
  23. else:
  24. client.HTTPDownloader.noPage(self, reason)
  25. def gotHeaders(self, headers):
  26. if self.status == "200":
  27. if headers.has_key("content-length"):
  28. self.totalbytes = int(headers["content-length"][0])
  29. else:
  30. self.totalbytes = 0
  31. self.currentbytes = 0.0
  32. return client.HTTPDownloader.gotHeaders(self, headers)
  33. def pagePart(self, packet):
  34. if self.status == "200":
  35. self.currentbytes += len(packet)
  36. if self.totalbytes and self.progress_callback:
  37. self.progress_callback(self.currentbytes, self.totalbytes)
  38. return client.HTTPDownloader.pagePart(self, packet)
  39. def pageEnd(self):
  40. return client.HTTPDownloader.pageEnd(self)
  41. class DownloadWithProgress:
  42. def __init__(self, url, outputfile, headers=None, limit=0, contextFactory=None, *args, **kwargs):
  43. self.limit = limit
  44. uri = urlparse.urlparse(url)
  45. scheme = uri.scheme
  46. host = uri.hostname
  47. port = uri.port if uri.port else 80
  48. path = uri.path
  49. if not headers:
  50. headers = {"user-agent":USER_AGENT}
  51. self.factory = HTTPProgressDownloader(url, outputfile, headers, *args, **kwargs)
  52. if scheme == "https":
  53. self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory())
  54. else:
  55. self.connection = reactor.connectTCP(host, port, self.factory)
  56. def start(self):
  57. return self.factory.deferred
  58. def stop(self):
  59. if self.connection:
  60. print "[stop]"
  61. self.connection.disconnect()
  62. def addProgress(self, progress_callback):
  63. print "[addProgress]"
  64. self.factory.progress_callback = progress_callback
  65. #####################################################################################################
  66. class DownloadWithProgressFragmented:
  67. def __init__(self, url, outputfile, headers = None, limit = 0, contextFactory=None, *args, **kwargs):
  68. self.url = url
  69. self.outputfile = outputfile
  70. self.base_url = "/".join(url.split("/")[:-1])+"/"
  71. self.headers = headers if headers else {"user-agent":"Enigma2"}
  72. self.limit = limit
  73. self.agent = kwargs["agent"] if "agent" in kwargs else None
  74. self.cookie = kwargs["cookie"] if "cookie" in kwargs else None
  75. self.deferred = defer.Deferred()
  76. #self.deferred.addCallback(self.start_download)
  77. def start_download(self):
  78. print "Start download"
  79. url = self.url
  80. while True:
  81. base_url = url.split("?")[0]
  82. base_url = "/".join(base_url.split("/")[:-1])+"/"
  83. try:
  84. r = requests.get(url,headers=self.headers)
  85. except Exception as e:
  86. #self.deferred.errback("Cannot open manifsest file - %s"%url)
  87. self.deferred.errback(e)
  88. if not r.content.startswith("#EXTM3U"):
  89. self.deferred.errback(Exception("Not valid manifest file - %s"%self.url))
  90. streams = re.findall(r"#EXT-X-STREAM-INF:.*?BANDWIDTH=(\d+).*?\n(.+?)$", r.content, re.IGNORECASE | re.MULTILINE)
  91. if streams:
  92. streams = sorted(streams, key=lambda item: int(item[0]), reverse=True)
  93. url = streams[0][1]
  94. if not url.startswith("http"):
  95. url = base_url + url
  96. try:
  97. r = requests.get(url, headers=self.headers)
  98. except Exception as e:
  99. self.deferred.errback(Exception("Cannot open manifsest file - %s"%url))
  100. if "#EXT-X-STREAM-INF" in r.content:
  101. continue
  102. else:
  103. break
  104. else:
  105. self.deferred.errback(Exception("No streams found for - %s"%url))
  106. self.base_url = base_url
  107. self.ts_list = re.findall(r"#EXTINF:([\d\.]+),.*?\n(.+?)$", r.content, re.IGNORECASE | re.MULTILINE)
  108. if not len(self.ts_list):
  109. self.deferred.errback(Exception("Cannot read fragment list in manifsest file - %s"%url))
  110. self.ts_num = 0
  111. self.type = "vod" if "#EXT-X-ENDLIST" in r.content else "live"
  112. self.currentbytes = 0.0
  113. self.totalbytes = -1
  114. self.currenttime = 0.0
  115. self.totaltime = sum(map(float,zip(*self.ts_list)[0]))
  116. self.ts_file = open(self.outputfile, "wb")
  117. self.download_fragment()
  118. def download_fragment(self):
  119. if self.ts_num>=len(self.ts_list):
  120. pass
  121. #print "Call later"
  122. reactor.callLater(10,self.update_manifest)
  123. reactor.callLater(10, self.download_fragment)
  124. else:
  125. print "Start fragment download"
  126. url = self.ts_list[self.ts_num][1]
  127. if not "://" in url:
  128. url = self.base_url+url
  129. self.d = client.getPage(url,headers = self.headers)
  130. self.d.addCallbacks(self.download_ok,self.download_err)
  131. def download_ok(self,content):
  132. content_length = len(content)
  133. self.currentbytes += content_length
  134. self.currenttime += float(self.ts_list[self.ts_num][0])
  135. self.totalbytes = self.currentbytes * self.totaltime / self.currenttime
  136. self.ts_num += 1
  137. #print "Fragment %s downloaded (%s)"%(self.ts_num,len(content))
  138. self.ts_file.write(content)
  139. self.progress_callback(self.currentbytes, self.totalbytes)
  140. if self.type == "vod":
  141. if self.ts_num >= len(self.ts_list) or (self.limit and self.currenttime>self.limit):
  142. self.ts_file.close()
  143. self.download_finished()
  144. else:
  145. self.download_fragment()
  146. else:
  147. if self.limit and self.currenttime>self.limit: # TODO
  148. self.ts_file.close()
  149. self.download_finished()
  150. else:
  151. self.download_fragment()
  152. def update_manifest(self):
  153. self.d2 = client.getPage(self.url, headers=self.headers)
  154. self.d2.addCallbacks(self.update_manifest_ok, self.update_manifest_err)
  155. def update_manifest_ok(self,content):
  156. print "Update manifest"
  157. ts_list = re.findall(r"#EXTINF:([\d\.]+),\n(.+?)$", content, re.IGNORECASE | re.MULTILINE)
  158. last_ts = self.ts_list[-1]
  159. found = False
  160. for ts in ts_list:
  161. if ts == last_ts:
  162. found = True
  163. elif found:
  164. print "Append %s"%ts[1]
  165. self.ts_list.append(ts)
  166. #reactor.callLater(5,self.download_fragment)
  167. def update_manifest_err(self,content):
  168. return
  169. def download_err(self,content):
  170. self.deferred.errback("Error while downloading %s"%self.ts_list[self.ts_num][1])
  171. def download_finished(self):
  172. self.totalbytes = self.currentbytes
  173. self.deferred.callback("Done")
  174. def start(self):
  175. reactor.callLater(1,self.start_download)
  176. return self.deferred
  177. def stop(self):
  178. self.deferred.errback() # TODO
  179. def addProgress(self, progress_callback):
  180. #print "[addProgress]"
  181. self.progress_callback = progress_callback
  182. #####################################################################################################
  183. def get_header(url,headers=None):
  184. headers = {"user-agent":USER_AGENT}
  185. r = requests.head(url,headers=headers)
  186. return r.headers
  187. def get_ext(mtype):
  188. stype = "http"
  189. if mtype in ("vnd.apple.mpegURL","application/x-mpegURL",'application/x-mpegurl',"application/vnd.apple.mpegurl"):
  190. return ".ts","hls"
  191. elif mtype in ("application/dash+xml"):
  192. return ".ts","dash" # TODO dash stream type could be different !
  193. elif mtype in ("video/mp4"):
  194. return ".mp4","http"
  195. elif mtype in ("video/MP2T","video/mp2t"):
  196. return ".ts","http"
  197. elif mtype in ("video/x-flv"):
  198. return ".flv","http"
  199. elif mtype in ("video/quicktime"):
  200. return ".mov","http"
  201. elif mtype in ("video/x-msvideo"):
  202. return ".avi","http"
  203. elif mtype in ("video/x-ms-wmv"):
  204. return ".wmv","http"
  205. elif mtype in ("video/x-matroska"):
  206. return ".mkv","http"
  207. else:
  208. return ".mp4","http"
  209. ##############################################
  210. def print_progress(currentbytes, totalbytes):
  211. progress = float(currentbytes)/float(totalbytes)*100
  212. print "%s (%i/%i)"%(progress,currentbytes,totalbytes)
  213. def download_ok(*args):
  214. print "Finished"
  215. stop()
  216. def download_err(e):
  217. print "Download Error %s"%e.getBriefTraceback()
  218. stop()
  219. def stop():
  220. reactor.stop()
  221. ###############################################
  222. def download_video_file(url,outputfile,headers=None):
  223. #output = stream["name"].replace("\\"," ").replace(":"," ").replace("|"," ")
  224. if not headers:
  225. headers = {"user-agent":"Enigma2"}
  226. try:
  227. h = get_header(url,headers=headers)
  228. mtype = h.get("content-type")
  229. ext,stream_type = get_ext(mtype)
  230. except:
  231. ext,stream_type = (".ts","hls")
  232. output = output+ext
  233. if stream_type == "hls":
  234. d = DownloadWithProgressFragmented(url, outputfile, headers={"user-agent":"Enigma2"})
  235. else:
  236. d = DownloadWithProgress(url, outputfile, headers={"user-agent":"Enigma2"})
  237. d.addProgress(print_progress)
  238. d.start().addCallback(download_ok).addErrback(download_err)
  239. return d
  240. if __name__ == "__main__":
  241. if len(sys.argv)>2:
  242. url= sys.argv[1]
  243. output = sys.argv[2]
  244. else:
  245. url = url2 = "http://walterebert.com/_/playground/videos/sintel-trailer.m3u8"
  246. url = "http://str5e.lattelecom.tv/mobile-vod/mp4:sv_ka_sapni_lv_hq.mp4/playlist.m3u8?resource_id=like_a_dream&auth_token=ISVlUKAuwHJEFFKlHQ9e/GVIoKExOwcu0ymn2AEyeOhn2BAwzmy+csGxUe9MeKooEdjS+xJfmP6XPCuqlZIpe4k/UfGAwn0BQ/chunklist_w1465619955.m3u8?resource_id=like_a_dream&auth_token=ISVlUKAuwHJEFFKlHQ9e%2FGVIoKExOwcu0ymn2AEyeOhn2BAwzmy+csGxUe9MeKooEdjS+xJfmP6XPCuqlZIpe4k%2FUfGAwn0BQ%2FuGk9oBsD4%3D"
  247. outputfile = "tmp\\test"
  248. #url = "https://r3---sn-bavc5ajvh-gpme.googlevideo.com/videoplayback?key=yt6&mime=video%2Fmp4&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Cratebypass%2Crequiressl%2Csource%2Cupn%2Cexpire&expire=1490986184&lmt=1490940183963773&dur=1302.639&itag=18&ratebypass=yes&mm=31&requiressl=yes&ipbits=0&upn=azFGj8gY02w&ip=85.254.87.15&pl=23&ei=aFDeWLzqDcn-dLC_gdAM&signature=083F353AC09CD98A70AD7D9438DD3C91C781166B.715456B9C35F040BDC4728CA76A0D1779B684A90&source=youtube&mv=m&mt=1490964451&ms=au&mn=sn-bavc5ajvh-gpme&gir=yes&clen=73596250&id=o-AGH9y-hWn1MtW1VzCyI_8XYYEWODsTDBZbfagQH3BrfQ&initcwndbps=4493750"
  249. #url = "http://techslides.com/demos/sample-videos/small.mp4"
  250. #url = "http://wx17.poiuytrew.pw/s/c507282042b1bf25e0b72c34a68426f3/hd_30/Jackie.2016.D.iTunes.BDRip.1080p_720.mp4"
  251. #url = "http://player.tvnet.lv/live/amlst:11/chunklist_w361981294_b528000.m3u8"
  252. #url = "http://vod-hls-uk-live.akamaized.net/usp/auth/vod/piff_abr_full_hd/a3e90e-b08ktytr/vf_b08ktytr_f9d55583-afc7-49bb-9bf4-d8f1ac99f56f.ism.hlsv2.ism/vf_b08ktytr_f9d55583-afc7-49bb-9bf4-d8f1ac99f56f.ism.hlsv2-audio=128000-video=5070000.m3u8"
  253. #url = "https://58174450afee9.streamlock.net/vod/mp4:_definst_/f/e/8e49fc32.mp4/playlist.m3u8?safwerwfasendtime=1490877870&safwerwfasstarttime=1490859339&safwerwfashash=hS2FfVZysQVazBQ6RJn1IhUevBkKxIF09Ly3BjfT43U="
  254. active = 2
  255. d1 = download_video_file(url,outputfile)
  256. reactor.run()