# -*- coding: utf-8 -*- # Server agnostic file operation similar to local files # Currently supported - local, ftp, http import os, sys, re import urllib2, ftplib def parse_url(url): #logger.info("Url: %s" % url) url = url.strip() import re m = re.search(r"^(?P\w+)://(?:(?P[^;]+);)?(?:(?P[^:@]+)[:|@])?(?:(?P[^@]+)@)?(?P[^/^:]+)?(?::(?P[^/]+))?(?P[/]?.*?)$", url, re.DOTALL | re.MULTILINE) if m: res = m.groupdict() else: res = {"domain": None, "host": None, "password": None, "path": url, "port": None, "protocol": "file", "user": None} return res class FileObject(object): def __init__(self, url, mode="r", encoding=""): self.url = url self.mode = mode self.encoding = encoding self.file = None p = parse_url(url) self.p = p if p["protocol"] == "file": self.file = __builtins__["open"](p["path"], mode) elif p["protocol"] == "ftp": import ftplib, urllib2 if "w" in mode or "a" in mode: self.file = ftplib.FTP(p["host"], p["user"], p["password"]) elif "r" in mode: self.file = urllib2.urlopen(url) elif p["protocol"] in ["http", "https"]: self.file = urllib2.urlopen(url) else: raise Exception("Not valid protocol") self.first = True def read(self, bytes=None): if self.p["protocol"] == "file": return self.file.read(bytes) elif self.p["protocol"] == "ftp": return self.file.read(bytes) elif self.p["protocol"] in ("http", "https"): return self.file.read(bytes) def write(self, data, bytes=None): if self.p["protocol"] == "file": return self.file.write(data) elif self.p["protocol"] == "ftp": from StringIO import StringIO if "w" in self.mode and self.first: cmd = "STOR " elif "a" in self.mode or "w" in self.mode: cmd = "APPE " else: raise Exception("Can not write in r mode") if "b" in self.mode: self.file.storbinary(cmd + self.p["path"], StringIO(data)) else: self.file.storlines(cmd + self.p["path"], StringIO(data)) self.first = False elif self.p["protocol"] in ("http", "https"): raise Exception("Write not allowed for http") else: raise Exception("Not valid protocol") def close(self): if self.p["protocol"] == "file": self.file.close() elif self.p["protocol"] == "ftp": if "w" in self.mode or "a" in self.mode: self.file.close() elif self.p["protocol"] in ("http", "https"): pass else: raise Exception("Not valid protocol") def open(url, mode="r", encoding=""): return FileObject(url, mode, encoding) def exists(url): p = parse_url(url) if p["protocol"] == "file": if isinstance(url, str): url = url.decode("utf8") return os.path.exists(url) elif p["protocol"] in ("ftp", "http", "https"): try: f = urllib2.urlopen(url) del f return True except: return False else: raise Exception("Not valid protocol") def isdir(url): p = parse_url(url) if p["protocol"] == "file": if isinstance(url, str): url = url.decode("utf8") return os.path.isdir(url) elif p["protocol"] in ("ftp"): file = ftplib.FTP(p["host"], p["user"], p["password"]) cmd = "MLSD "+p["path"] lines = [] file.retrbinary(cmd, lines.append) file.close() return False if lines and "No such file or directory" in lines[0] else True else: raise Exception("Not valid protocol") def mkdir(url): p = parse_url(url) if p["protocol"] == "file": os.mkdir(url) elif p["protocol"] in ("ftp"): file = ftplib.FTP(p["host"], p["user"], p["password"]) file.mkd(p["path"]) file.close() else: raise Exception("Not valid protocol") def rmdir(url): p = parse_url(url) if p["protocol"] == "file": os.rmdir(url) elif p["protocol"] in ("ftp"): file = ftplib.FTP(p["host"], p["user"], p["password"]) file.rmd(p["path"]) file.close() else: raise Exception("Not valid protocol") def remove(url): p = parse_url(url) if p["protocol"] == "file": os.remove(url) elif p["protocol"] in ("ftp"): file = ftplib.FTP(p["host"], p["user"], p["password"]) file.delete(p["path"]) file.close() else: raise Exception("Not valid protocol") def rename(url, url2): p = parse_url(url) p2 = parse_url(url2) if p["protocol"] == "file": os.rename(url, url2) elif p["protocol"] in ("ftp"): file = ftplib.FTP(p["host"], p["user"], p["password"]) file.rename(p["path"], p2["path"]) file.close() else: raise Exception("Not valid protocol") def listdir(url): p = parse_url(url) if p["protocol"] == "file": return os.listdir(url) elif p["protocol"] in ("ftp"): cmd = "MLSD "+p["path"] file = ftplib.FTP(p["host"], p["user"], p["password"]) lines = [] file.retrlines(cmd, lines.append) file.close() return lines else: raise Exception("Not valid protocol") def join(*arg): n = len(arg) if n == 0: return "" elif n == 1: return arg[0] p = parse_url(arg[0]) sep = os.path.sep if p["protocol"] == "file" else "/" if "/" in arg[0]: sep = "/" elif "\\" in arg[0]: sep = "\\" path = arg[0] for i in xrange(1, n): if arg[i-1][-1] == sep: path = path + arg[i] else: path = path + sep + arg[i] return path def encoding(url): p = parse_url(url) if p["protocol"] == "file": enc = sys.getfilesystemencoding() #enc = "utf8" # TODO else: enc = "utf8" return enc def make_fname(name): #"\[.+?\]" "[/\n\r\t,:]" name = re.sub(r"[/\n\r\t,:/\\]", " ", name) name = re.sub("['""\?\*<>]","", name) name = re.sub(r"(\[[^\[]*\])", "", name) name = name.replace(""","") name = name.strip() return name def encode(url): p = parse_url(url) if p["protocol"] == "file": # unicode if not isinstance(url, unicode): url = url.decode("utf8") else: if isinstance(url, unicode): url = url.encode("utf8") return url if __name__ == "__main__": res = join("aaa", "bbb") #url = "smb://user:xxxx@192.168.77.197/hdd/test" url = "ftp://user:xxxx@home.blue.lv:21/hdd/test" url = "ftp://user:xxxx@192.168.77.197:21/hdd/test" mkdir("ftp://user:xxxx@192.168.77.197:21/hdd/movie/AAA") res = listdir("ftp://user:xxxx@home.blue.lv/hdd/movie/") print res rmdir("ftp://user:xxxx@192.168.77.197:21/hdd/movie/AAA") rename("ftp://user:xxxx@192.168.77.197:21/hdd/test", "ftp://user:xxxx@192.168.77.197:21/hdd/test2") remove("ftp://user:xxxx@192.168.77.197:21/hdd/test2") e = isdir("ftp://user:xxxx@192.168.77.197:21/hdd/movies/TV2") print e f = open(url, "wb") f.write("6.rinda\n") f.write("7.rinda\n") f.close() f = open(url, "rb") res = f.read() print res f.close() e = exists(url) print e e = exists(url+"2") print e