#!/usr/bin/python #@+leo-ver=4 #@+node:@file gmailfs.py #@@first # # Copyright (C) 2004 Richard Jones # Copyright (C) 2010 Dave Hansen # # GmailFS - Gmail Filesystem Version 0.8.0 # This program can be distributed under the terms of the GNU GPL. # See the file COPYING. # # TODO: # Problem: a simple mkdir ends up costing at least 3 server writes: # 1. create directory entry # 2. create inode # 3. create first data block # It would be greate if files below a certain size (say 64k or something) # could be inlined and just stuck as an attachment inside the inode. # It should not be too big or else it will end up making things like # stat() or getattr() much more expensive # """ GmailFS provides a filesystem using a Google Gmail account as its storage medium """ #@+others #@+node:imports import pprint import fuse import imaplib import email from email import encoders from email.mime.multipart import MIMEMultipart from email.MIMEText import MIMEText from email.mime.base import MIMEBase from fuse import Fuse import os from errno import * from stat import * from os.path import abspath, expanduser, isfile fuse.fuse_python_api = (0, 2) import thread import quopri from lgconstants import * import sys,traceback,re,string,time,tempfile,array,logging,logging.handlers #imaplib.Debug = 4 #@-node:imports # Globals DefaultUsername = 'defaultUser' DefaultPassword = 'defaultPassword' DefaultFsname = 'gmailfs' References={} IMAPBlockSize = 1024 InlineInodeMax = 64 * 1024 SystemConfigFile = "/etc/gmailfs/gmailfs.conf" UserConfigFile = abspath(expanduser("~/.gmailfs.conf")) GMAILFS_VERSION = '4' PathStartDelim = '__a__' PathEndDelim = '__b__' FileStartDelim = '__c__' FileEndDelim = '__d__' LinkStartDelim = '__e__' LinkEndDelim = '__f__' MagicStartDelim = '__g__' MagicEndDelim = '__h__' InodeSubjectPrefix = 'inode_msg' DirentSubjectPrefix = 'dirent_msg' InodeTag ='i' DevTag = 'd' NumberLinksTag = 'k' FsNameTag = 'q' ModeTag = 'e' UidTag = 'u' GidTag = 'g' SizeTag = 's' AtimeTag = 'a' MtimeTag = 'm' CtimeTag = 'c' BlockSizeTag = 'z' VersionTag = 'v' RefInodeTag = 'r' FileNameTag = 'n' PathNameTag = 'p' LinkToTag = 'l' NumberQueryRetries = 1 regexObjectTrailingMB = re.compile(r'\s?MB$') def log_debug(str): log.debug(str) #str += "\n" #sys.stderr.write(str) return def log_debug1(str): log.info(str) #str += "\n" #sys.stderr.write(str) return def log_debug2(str): #log.debug(str) #str += "\n" #sys.stderr.write(str) return def log_debug3(str): return def log_imap(str): #log_info("IMAP: " + str) log_debug2("IMAP: " + str) def log_info(str): log.info(str) #str += "\n" #sys.stderr.write(str) return def log_warning(str): log.warning(str) #str += "\n" #sys.stderr.write(str) return def msg_add_payload(msg, payload, filename=None): attach_part = MIMEBase('file', 'attach') attach_part.set_payload(payload) if filename != None: attach_part.add_header('Content-Disposition', 'attachment; filename="%s"' % filename) encoders.encode_base64(attach_part) msg.attach(attach_part) # This probably doesn't need to be handed the fsNameVar # and the username def mkmsg(username, fsNameVar, subject, preamble, attach = ""): msg = MIMEMultipart() log_debug2("mkmsg('%s', '%s', '%s', '%s',...)" % (username, fsNameVar, subject, preamble)) msg['Subject'] = subject msg['To'] = username msg['From'] = username msg.preamble = preamble if len(attach): log_debug("attaching %d byte file contents" % len(attach)) msg_add_payload(msg, attach) log_debug3("mkmsg() after subject: '%s'" % (msg['Subject'])) return msg def imap_append(info, imap, msg): #gmsg = libgmail.GmailComposedMessage(username, subject, body) log_debug2("imap_append(%s, '%s')" % (info, str(imap))) log_debug3("entire message: ->%s<-" % str(msg)) now = imaplib.Time2Internaldate(time.time()) rsp, data = imap.append(fsNameVar, "", now, str(msg)) log_imap("append for '%s': rsp,data: '%s' '%s'" % (info, rsp, data)) clear_query_cache() if rsp != "OK": return -1 # data looks like this: '['[APPENDUID 631933985 286] (Success)']' msgid = int((data[0].split()[2]).replace("]","")) msg.uid = msgid log_debug("imap msgid: '%d'" % msgid) return msgid def trashMessage(imap, msg): uid = msg.uid trashMessageByUid(imap, uid) def trashMessageByUid(imap, uid): log_imap("trashing message number: " + str(uid)) imap.uid("STORE", uid, '+FLAGS', '\\Deleted') global uid_to_subj_cache try: del uid_to_subj_cache[uid] except: foo = 1 # this is OK because the msg may neve have # been cached return def _addLoggingHandlerHelper(handler): """ Sets our default formatter on the log handler before adding it to the log object. """ handler.setFormatter(defaultLogFormatter) log.addHandler(handler) def GmailConfig(fname): import ConfigParser cp = ConfigParser.ConfigParser() global References global DefaultUsername, DefaultPassword, DefaultFsname global NumberQueryRetries if cp.read(fname) == []: log_warning("Unable to read configuration file: " + str(fname)) return sections = cp.sections() if "account" in sections: options = cp.options("account") if "username" in options: DefaultUsername = cp.get("account", "username") if "password" in options: DefaultPassword = cp.get("account", "password") else: log.error("Unable to find GMail account configuration") if "filesystem" in sections: options = cp.options("filesystem") if "fsname" in options: DefaultFsname = cp.get("filesystem", "fsname") else: log_warning("Using default file system (Dangerous!)") if "logs" in sections: options = cp.options("logs") if "level" in options: level = cp.get("logs", "level") log.setLevel(logging._levelNames[level]) if "logfile" in options: logfile = abspath(expanduser(cp.get("logs", "logfile"))) log.removeHandler(defaultLoggingHandler) _addLoggingHandlerHelper(logging.handlers.RotatingFileHandler(logfile, "a", 5242880, 3)) if "references" in sections: options = cp.options("references") for option in options: record = cp.get("references",option) fields = record.split(':') if len(fields)<1 or len(fields)>3: log_warning("Invalid reference '%s' in configuration." % (record)) continue reference = reference_class(*fields) References[option] = reference class reference_class: def __init__(self,fsname,username=None,password=None): self.fsname = fsname if username is None or username == '': self.username = DefaultUsername else: self.username = username if password is None or password == '': self.password = DefaultPassword else: self.password = password # This ensures backwards compatability where # old filesystems were stored with 7bit encodings # but new ones are all quoted printable def fixQuotedPrintable(body): # first remove headers newline = body.find("\r\n\r\n") if newline >= 0: body = body[newline:] fixed = body if re.search("Content-Transfer-Encoding: quoted",body): fixed = quopri.decodestring(body) # Map unicode return fixed.replace('\u003d','=') def psub(s): if len(s) == 0: return ""; return "SUBJECT \""+s+"\"" query_cache = {} uid_to_subj_cache = {} def clear_query_cache(): global query_cache log_debug2("clearing query cache (had %d entries)" % (len(query_cache))) query_cache = {} return def _getMsguidsByQuery(imap, queries): tries = 0 queries.append(str(FsNameTag + "=" + MagicStartDelim + fsNameVar + MagicEndDelim)) # this is *REALLY* sensitive, at least on gmail # Don't put any extra space in it anywhere, or you # will be sorry # 53:12.12 > MGLK6 SEARCH (SUBJECT "foo=bar" SUBJECT "bar=__fo__o__") queryString = '(SUBJECT "' + string.join(queries, '" SUBJECT "') + '")' global query_cache if queryString in query_cache: ret = query_cache[queryString] log_debug2("query_cache hit! '%s' -> '%s'" % (queryString, string.join(ret," "))) return ret # make sure mailbox is selected log_imap("SEARCH query: '"+queryString+"'") try: resp, msgids_list = imap.uid("SEARCH", None, queryString) except: exit(-1) msgids = msgids_list[0].split(" ") log_debug2("IMAP search resp: %s msgids len: %d" % (resp, len(msgids))) ret = [] for msgid in msgids: log_debug2("IMAP search result msg_uid: '%s'" % str(msgid)) if len(str(msgid)) > 0: ret = msgids break # can I store arrays of arrays? query_cache[queryString] = ret return ret def getSingleMsguidByQuery(imap, q): msgids = _getMsguidsByQuery(imap, q) nr = len(msgids) if nr != 1: qstr = string.join(q, " ") # this is debug because it's normal to have non-existent files log_debug("could not find inode for query: '%s' (found %d)" % (qstr, nr)) return -1; log_debug2("getSingleMsguidByQuery('%s') ret: '%s' nr: %d" % (string.join(q," "), msgids[0], nr)) return int(msgids[0]) def fetchRFC822ForMsguid(imap, msgid): if msgid == None: return None log_imap("fetch(msgid=%s)" % (msgid)) #resp, data = imap.fetch(msgid, '(RFC822)') resp, data = imap.uid("FETCH", msgid, '(RFC822)') data = data[0][1] log_debug2("fetch msgid: '%s' resp: '%s' data: %d bytes" % (str(msgid), resp, len(data))) return data def getSingleMessageByQuery(imap, q): log_debug("getSingleMessageByQuery()") msgid = getSingleMsguidByQuery(imap, q) if msgid == -1: return None msg_str = fetchRFC822ForMsguid(imap, msgid) if msg_str == None: return None msg = email.message_from_string(msg_str) msg.uid = msgid return msg def _pathSeparatorEncode(path): s1 = re.sub("/","__fs__",path) s2 = re.sub("-","__mi__",s1) return re.sub("\+","__pl__",s2) def _pathSeparatorDecode(path): s1 = re.sub("__fs__","/",path) s2 = re.sub("__mi__","-",path) return re.sub("__pl__","+",path) def _logException(msg): traceback.print_exc(file=sys.stderr) log.exception(msg) #@+node:class GmailInode class GmailInode: """ Class used to store gmailfs inode details """ #@+node:__init__ def __init__(self, dirent_msg, inode_msg, imap): try: self.version = 2 self.ino = 0 self.mode = 0 self.dev = 0 self.nlink = 0 self.uid = 0 self.gid = 0 self.size = 0 self.atime = 0 self.mtime = 0 self.ctime = 0 self.blocksize = DefaultBlockSize self.imap = imap self.dirent_msg = dirent_msg self.inode_msg = inode_msg self.xattr = {} self.setInode() except: _logException("got exception when getmessages1") #@-node:__init__ #@+node:getinodeMsg def getinodeMsg(self,inodeNumber): """ Get Gmail message handle for the message containing inodeNumber """ msg = getSingleMessageByQuery(self.imap, [InodeTag+'='+inodeNumber+'']) if msg == None: log_warning("unable to find node nr: '%s'\n" % (inodeNumber)) time.sleep(100) return None inode.xattr = self.decode_xattr(inode.inode_msg) log_debug("getinodeMsg() nr: '%s' found msg: was '%d' bytes" % (inodeNumber, len(str(msg)))) return msg #@-node:getinodeMsg def update(self): """ Sync's the state of this inode back to the users gmail account """ timeString = str(int(time.time())) subject = ( InodeSubjectPrefix+ " "+VersionTag+"="+GMAILFS_VERSION+ " "+InodeTag+"="+str(self.ino)+ " "+DevTag+"="+str(self.dev)+ " "+NumberLinksTag+"="+str(self.nlink)+ " "+FsNameTag+"="+MagicStartDelim+ fsNameVar +MagicEndDelim ) body = (ModeTag+"="+str(self.mode)+ " "+UidTag+"="+str(self.uid)+ " "+GidTag+"="+str(self.gid)+ " "+SizeTag+"="+str(self.size)+ " "+AtimeTag+"="+str(self.atime)+ " "+MtimeTag+"="+timeString+ " "+CtimeTag+"="+str(self.ctime)+ " "+BlockSizeTag+"="+str(self.blocksize) ) msg = mkmsg("dave@sr71.net", fsNameVar, subject, body) for attr in self.xattr: value = self.xattr[attr] payload_name = 'xattr-'+attr log_debug1("adding xattr payload named '%s': '%s'" % (payload_name, value)) msg_add_payload(msg, value, payload_name) msgid = imap_append("update inode", self.imap, msg) if msgid > 0: log_debug("update() Sent subject '"+subject+"' ok, about to call getinodeMsg()") if (self.inode_msg): log_debug("trashing old inode:"+str(self.inode_msg['Subject'])) trashMessage(self.imap, self.inode_msg) # it was kinda silly to go fetch this when we *just* # created it in the same function up there ^^ #self.inode_msg = self.getinodeMsg(str(self.ino)) self.inode_msg = msg else: e = OSError("Couldnt send mesg:"+msg['Subject']) e.errno = ENOSPC raise e # Uh oh. Does this properly truncate data blocks that are no # longer in use? def fill_xattrs(self): log_debug1("fill_xattrs()") for part in self.inode_msg.get_payload(): log_info("fill_xattrs() loop") fname = part.get_filename(None) log_info("fill_xattrs() fname: '%s'" % (str(fname))) if fname == None: continue m = re.match('xattr-(.*)', fname) if m == None: continue xattr_name = m.group(1) log_info("fill_xattrs() xattr_name: '%s'" % (xattr_name)) self.xattr[xattr_name] = part.get_payload(decode=True) def setInode(self): """ Setup the inode instances members from the gmail inode message """ try: subject = self.inode_msg['Subject'].replace('\u003d','=') if self.inode_msg.is_multipart(): body = self.inode_msg.preamble log_debug("message was multipart, reading body from preamble") else: # this is a bug log_debug("message was single part") log_debug1("body: ->%s<-" % body) body = fixQuotedPrintable(body) log_debug3("setting inode from subject:"+subject) m = re.match((InodeSubjectPrefix+' '+ VersionTag+'=(.*) '+ InodeTag+'=(.*) '+ DevTag+'=(.*) '+ NumberLinksTag+'=(.*) '+ FsNameTag+'='+MagicStartDelim+'(.*)'+MagicEndDelim), subject) self.version = int(m.group(1)) self.ino = int(m.group(2)) self.dev = int(m.group(3)) self.nlink = int(m.group(4)) #quotedEquals = "=(?:3D)?(.*)" quotedEquals = "=(.*)" m = re.search(re.compile( ModeTag+quotedEquals+' ?'+UidTag+quotedEquals+' ?'+ GidTag+quotedEquals+' ?'+ SizeTag+quotedEquals+' ?'+ AtimeTag+quotedEquals+' ?'+ MtimeTag+quotedEquals+' ?'+ CtimeTag+quotedEquals+' ?'+ BlockSizeTag+quotedEquals, re.DOTALL),body) self.mode = int(m.group(1)) self.uid = int(m.group(2)) self.gid = int(m.group(3)) self.size = int(m.group(4)) self.atime = int(m.group(5)) self.mtime = int(m.group(6)) self.ctime = int(m.group(7)) self.blocksize = int(m.group(8)) self.fill_xattrs() except: _logException("got exception when setInode") self.ino = None def unlink(self, path): """ Delete this inode and all of its data blocks """ log_debug1("unlink path:"+path+" with nlinks:"+str(self.nlink)) self.nlink-=1 if self.mode & S_IFDIR: log_debug("unlinking dir") self.nlink-=1 else: log_debug("unlinking file") trashMessage(self.imap, self.dirent_msg) if self.nlink<1: log_debug("deleting inode or block nlink:"+str(self.nlink)) trashMessage(self.imap, self.inode_msg) subject = 'b='+str(self.ino)+'' log_info("unlink _getSingleMsguidByQuery('%s')" % subject) msgids = _getMsguidsByQuery(self.imap, [subject]) for msgid in msgids: trashMessageByUid(self.imap, msgid) else: log_debug("about to update inode") self.update() log_debug("not deleting inode or block nlink:"+str(self.nlink)) #@-node:class GmailInode #@+node:class OpenGmailFile class OpenGmailFile: """ Class holding any currently open files, includes cached instance of the last data block retrieved """ def __init__(self, inode): self.inode = inode self.tmpfile = None self.blocksRead = 0 self.needsWriting = 0 self.blocksize = inode.blocksize self.buffer = list(" "*self.blocksize) self.currentOffset = -1 self.lastBlock = 0 self.lastBlockRead = -1 self.lastBlockBuffer = [] def close(self): """ Closes this file by committing any changes to the users gmail account """ if self.needsWriting: self.commitToGmail() def write(self,buf,off): """ Write data to file from buf, offset by off bytes into the file """ log_debug2("write buf: '%s' off: %d self.currentOffset: %d\n" % (buf, off, self.currentOffset)) buflen = len(buf) towrite = buflen #if self.currentOffset == -1 or offself.currentOffset+self.blocksize: # self.commitToGmail() # self.currentOffset = (off/self.blocksize)*self.blocksize+(off/self.blocksize) # self.buffer = self.readFromGmail(self.currentOffset/self.blocksize,1) if self.currentOffset == -1 or offself.currentOffset: self.commitToGmail() self.currentOffset = off; self.buffer = self.readFromGmail(self.currentOffset/self.blocksize,1) currentBlock = self.currentOffset/self.blocksize written = 0 while towrite>0: thiswrote = min(towrite,min(self.blocksize-(self.currentOffset%self.blocksize),self.blocksize)) log_debug2("wrote "+str(thiswrote)+" bytes off:"+str(off)+" self.currentOffset:"+str(self.currentOffset)) self.buffer[self.currentOffset%self.blocksize:] = buf[written:written+thiswrote] towrite -= thiswrote written += thiswrote self.currentOffset += thiswrote self.lastBlock = currentBlock log_debug2("write() setting needsWriting") self.needsWriting = 1 if self.currentOffset / self.blocksize > currentBlock: self.commitToGmail() currentBlock += 1 if towrite > 0: self.buffer = self.readFromGmail(currentBlock,1) if off+buflen > self.inode.size: self.inode.size = off+buflen return buflen def commitToGmail(self): """ Send any unsaved data to users gmail account as an attachment """ log_debug("commitToGmail() self.needsWriting: %d" % self.needsWriting) if not self.needsWriting: return 1 #a = self.inode.ga subject = ('b='+str(self.inode.ino)+ ' x='+str(self.lastBlock)+ ' '+FsNameTag+'='+MagicStartDelim+ fsNameVar +MagicEndDelim ) tmpf = tempfile.NamedTemporaryFile() arr = array.array('c') arr.fromlist(self.buffer) log_debug("wrote contents to tmp file: ->"+arr.tostring()+"<-") tmpf.write(arr.tostring()) tmpf.flush() msg = mkmsg(username, fsNameVar, subject, fsNameVar, arr.tostring()) msgid = imap_append("commit data blocks", self.inode.imap, msg) log_debug("commitToGmail() finished, rsp: '%s'" % str(msgid)) if msgid > 0: log_debug("Sent write commit ok") self.needsWriting = 0 self.inode.update() tmpf.close() return 1 else: log.error("Sent write commit failed") tmpf.close() return 0 def read(self,readlen,offset): """ Read readlen bytes from an open file from position offset bytes into the files data """ readlen = min(self.inode.size-offset,readlen) outbuf = list(" "*readlen) toread = readlen; upto = 0; while toread>0: readoffset = (offset+upto)%self.blocksize thisread = min(toread,min(self.blocksize-(readoffset%self.blocksize),self.blocksize)) outbuf[upto:] = self.readFromGmail((offset+upto)/self.blocksize,0)[readoffset:readoffset+thisread] upto+=thisread toread-=thisread log_debug2("still to read: "+str(toread)+" upto now: " + str(upto)) log_debug3("joined outbuf: ->%s<-" % string.join(outbuf, "")) return outbuf def readFromGmail(self,readblock,deleteAfter): """ Read data block with block number 'readblock' for this file from users gmail account, if 'deleteAfter' is true then the block will be removed from Gmail after reading """ log_debug2("readFromGmail() about to try and find inode:"+str(self.inode.ino)+" blocknumber:"+str(readblock)) if self.lastBlockRead == readblock: log_info("hit self.lastBlockRead cache") contentList = list(" "*self.blocksize) contentList[0:] = self.lastBlockBuffer return contentList q1 = 'b='+str(self.inode.ino) q2 = 'x='+str(readblock) msg = getSingleMessageByQuery(self.inode.imap, [ q1, q2 ]) if msg == None: log_debug2("readFromGmail(): file has no blocks, returning empty contents (%s %s)" % (q1, q2)) return list(" "*self.blocksize) log_debug2("got msg with subject:"+msg['Subject']) for part in msg.walk(): log_debug2("message part.get_content_maintype(): '%s'" % part.get_content_maintype()) if part.get_content_maintype() == 'multipart': continue #if part.get('Content-Disposition') is None: # continue log_debug2("message is multipart") a = part.get_payload(decode = True) log_debug3("part payload has len: %d asstr: '%s'" % (len(a), str(a))) log_debug3("after loop, a: '%s'" % str(a)) a = list(a) if deleteAfter: trashMessage(self.inode.imap, msg) self.lastBlockRead = readblock self.lastBlockBuffer = a contentList = list(" "*self.blocksize) contentList[0:] = a return contentList #@-node:class OpenGmailFile #@+node:class Gmailfs class Gmailfs(Fuse): #@ @+others #@+node:__init__ def __init__(self, extraOpts, mountpoint, *args, **kw): Fuse.__init__(self, *args, **kw) self.fuse_args.mountpoint = mountpoint self.fuse_args.setmod('foreground') self.optdict = extraOpts log_debug("Mountpoint: %s" % mountpoint) # obfuscate sensitive fields before logging #loggableOptdict = self.optdict.copy() #loggableOptdict['password'] = '*' * 8 #log_info("Named mount options: %s" % (loggableOptdict,)) # do stuff to set up your filesystem here, if you want self.openfiles = {} self.inodeCache = {} global DefaultBlockSize global fsNameVar global password global username DefaultBlockSize = 5*1024*1024 fsNameVar = DefaultFsname password = DefaultPassword username = DefaultUsername # options_required = 1 # if self.optdict.has_key("reference"): # try: # reference = References[self.optdict['reference']] # username = reference.username # password = reference.password # fsNameVar = reference.fsname # except: # log.error("Invalid reference supplied. Using defaults.") # else: # options_required = 0 # # if not self.optdict.has_key("username"): # if options_required: # log_warning('mount: warning, should mount with username=gmailuser option, using default') # else: # username = self.optdict['username'] # # if not self.optdict.has_key("password"): # if options_required: # log_warning('mount: warning, should mount with password=gmailpass option, using default') # else: # password = self.optdict['password'] # # if not self.optdict.has_key("fsname"): # if options_required: # log_warning('mount: warning, should mount with fsname=name option, using default') # else: # fsNameVar = self.optdict['fsname'] # # if self.optdict.has_key("blocksize"): # DefaultBlockSize = int(self.optdict['blocksize']) self.imap = imaplib.IMAP4_SSL("imap.gmail.com", 993);#libgmail.GmailAccount(username, password) #self.imap.debug = 4 username = "xxxxxxxxxxxxxxxxxxxxxxxxxx" password = "xxxxxxxxxxxxx" if username.find("@")<0: username = username+"@gmail.com" self.imap.login(username, password) # This select() can be done read-only # might be useful for implementing "mount -o ro" resp, data = self.imap.select(fsNameVar) log_debug("folder select '%s' resp: '%s' data: '%s'" % (fsNameVar, resp, data)) if resp == "NO": resp, data = self.imap.create(fsNameVar) log_debug("create '%s' resp: '%s' data: '%s'" % (fsNameVar, resp, data)) resp, data = self.imap.select(fsNameVar) log_debug("select2 '%s' resp: '%s' data: '%s'" % (fsNameVar, resp, data)) return log_info("Connected to gmail") #resp, data = self.imap.list() #log_info("list resp: " + resp) #for mbox in data: # log_info("mbox: " + mbox) #log_info("done listing mboxes") #FIXME # we should probably make a mkfs command to # make the root inode. We should probably # also make it search out and clear all # messages with the given label if 1 == 0: log_info("deleting existing messages") resp, msgids = self.imap.uid("SEARCH", 'ALL') for num in msgids[0].split(): log_info("deleting uid: '%s'" % (num)) self.imap.uid("STORE", num, '+FLAGS', '\\Deleted') log_info("done deleting existing messages") #thread.start_new_thread(self.mythread, ()) pass #@-node:__init__ #@+node:mythread def mythread(self): """ The beauty of the FUSE python implementation is that with the python interp running in foreground, you can have threads """ log_debug("mythread: started") #while 1: # time.sleep(120) # print "mythread: ticking" #@-node:mythread #@+node:attribs flags = 1 #@-node:attribs class GmailStat(fuse.Stat): def __init__(self): self.st_mode = 0 self.st_ino = 0 self.st_dev = 0 self.st_nlink = 0 self.st_uid = 0 self.st_gid = 0 self.st_size = 0 self.st_atime = 0 self.st_mtime = 0 self.st_ctime = 0 self.st_blocks = 0 global IMAPBlockSize self.st_blksize = IMAPBlockSize self.st_rdev = 0 #@+node:getattr def getattr(self, path): st = Gmailfs.GmailStat(); log_debug1("getattr('%s')" % (path)) #st_mode (protection bits) #st_ino (inode number) #st_dev (device) #st_nlink (number of hard links) #st_uid (user ID of owner) #st_gid (group ID of owner) #st_size (size of file, in bytes) #st_atime (time of most recent access) #st_mtime (time of most recent content modification) #st_ctime (time of most recent content modification or metadata change). if path == '/': inode = self.getinode(path) if not inode: log_info("creating root inode") self._mkfileOrDir("/",None,S_IFDIR|S_IRUSR|S_IXUSR|S_IWUSR|S_IRGRP|S_IXGRP|S_IXOTH|S_IROTH,-1,1,2) inode = self.getinode(path) else: inode = self.getinode(path) if inode: if log.isEnabledFor(logging.DEBUG): log_debug("inode "+str(inode)) st.st_mode = inode.mode st.st_ino = inode.ino st.st_dev = inode.dev st.st_nlink = inode.nlink st.st_uid = inode.uid st.st_gid = inode.gid st.st_size = inode.size st.st_atime = inode.atime st.st_mtime = inode.mtime st.st_ctme = inode.ctime # statTuple = (inode.mode,inode.ino,inode.dev,inode.nlink,inode.uid, # inode.gid,inode.size,inode.atime,inode.mtime, # inode.ctime) if log.isEnabledFor(logging.DEBUG): log_debug("statsTuple "+str(st)) return st else: e = OSError("No such file"+path) e.errno = ENOENT raise e #@-node:getattr #@+node:readlink def readlink(self, path): log_debug1("readlink: path='%s'" % path) inode = self.getinode(path) if not (inode.mode & S_IFLNK): e = OSError("Not a link"+path) e.errno = EINVAL raise e log_debug("about to follow link in body:"+inode.msg.as_string()) body = fixQuotedPrintable(inode.msg.as_string()) m = re.search(LinkToTag+'='+LinkStartDelim+'(.*)'+ LinkEndDelim,body) return m.group(1) #@-node:readlink def msgid_to_subj(self, msgid): global uid_to_subj_cache if msgid in uid_to_subj_cache: return uid_to_subj_cache[msgid] log_imap("fetching subject for msgid: '%s'" %(msgid)) resp, msg_data = self.imap.uid("FETCH", msgid, '(BODY[HEADER.FIELDS (SUBJECT)])') ret = "" for response_part in msg_data: if isinstance(response_part, tuple): #log_info("msgid_to_subj response_part: '%s'" % str(response_part)) ret = response_part[1] break uid_to_subj_cache[msgid] = ret return ret def msgid_to_body(self, msgid): log_imap("fetching body for msgid: '%s'" %(msgid)) resp, msg_data = self.imap.uid("FETCH", msgid, '(BODY.PEEK[TEXT])') for response_part in msg_data: if isinstance(response_part, tuple): #log_info("msgid_to_body response_part: '%s'" % str(response_part)) return response_part[1] #@+node:readdir def readdir(self, path, offset): log_debug1("readdir: path='%s'" % path) try: log_debug2("at top of readdir"); log_debug2("getting dir "+path) fspath = _pathSeparatorEncode(path) log_debug2("querying for:"+''+PathNameTag+'='+PathStartDelim+ fspath+PathEndDelim) # FIX need to check if directory exists and return error if it doesnt, actually # this may be done for us by fuse q = ''+PathNameTag+'='+PathStartDelim+fspath+PathEndDelim msgids = _getMsguidsByQuery(self.imap, [q]) log_debug2("readdir _getSingleMsguidByQuery('%s')" % q) log_debug2("got folder ") lst = [] for dirlink in ".", "..": lst.append(dirlink) for msgid in msgids: #log_debug("thread.summary is " + thread.snippet) subject = self.msgid_to_subj(msgid) m = re.search(FileNameTag+'='+FileStartDelim+'(.*)'+ FileEndDelim, subject) if (m): # Match succeeded, we got the whole filename. log_debug("Used summary for filename") filename = m.group(1) else: # Filename was too long, have to fetch message. log_debug("Long filename, had to fetch message") body = fixQuotedPrintable(self.msgid_to_body(msgid)) m = re.search(FileNameTag+'='+FileStartDelim+'(.*)'+ FileEndDelim, body) filename = m.group(1) log_debug("readdir('%s') found file: '%s'" % (path, filename)) # this test for length is a special case hack for the root directory to prevent ls /gmail_root # returning "". This is hack is requried due to adding modifiable root directory as an afterthought, rather # than designed in at the start. if len(filename)>0: lst.append(filename) except: _logException("got exception when getmessages2") lst = None #return map(lambda x: (x,0), lst) for r in lst: yield fuse.Direntry(r) #@-node:getdir #@+node:unlink def unlink(self, path): log_debug1("unlink called on:"+path) try: inode = self.getinode(path) inode.unlink(path) #del self.inodeCache[path] # this cache flushing in unfortunate but currently necessary # to avoid problems with hard links losing track of # number of the number of links self.inodeCache = {} return 0 except: _logException("Error unlinking file"+path) e = OSError("Error unlinking file"+path) e.errno = EINVAL raise e #@-node:unlink #@+node:rmdir def rmdir(self, path): log_debug1("rmdir called on:"+path) #this is already checked before rmdir is even called #dirlist = self.getdir(path) #if len(dirlist)>0: # e = OSError("directory not empty"+path) # e.errno = ENOTEMPTY # raise e inode = self.getinode(path) inode.unlink(path) #del self.inodeCache[path] # this cache flushing in unfortunate but currently necessary # to avoid problems with hard links losing track of # number of the number of links self.inodeCache = {} # update number of links in parent directory ind = string.rindex(path,'/') parentdir = path[:ind] log_debug("about to rmdir with parentdir:"+parentdir) if len(parentdir)==0: parentdir = "/" parentdirinode = self.getinode(parentdir) if parentdirinode == None: log_error("rmdir() unable to find parent inode: '%s'" % (parentdir)) parentdirinode.nlink-=1 parentdirinode.update() del self.inodeCache[parentdir] return 0 #@-node:rmdir #@+node:symlink def symlink(self, path, path1): log_debug1("symlink: path='%s', path1='%s'" % (path, path1)) self._mkfileOrDir(path1,path,S_IFLNK|S_IRWXU|S_IRWXG|S_IRWXO,-1,0,1) #@-node:symlink #@+node:rename def rename(self, path, path1): log_debug1("rename from:"+path+" to:"+path1) msg = self.fetch_msg_for_path(path) ind = string.rindex(path1,'/') log_debug("ind:"+str(ind)) dirpath = path1[:ind] if len(dirpath)==0: dirpath = "/" name = path1[ind+1:] log_debug("dirpath:"+dirpath+" name:"+name) fspath = _pathSeparatorEncode(dirpath) m = re.match(DirentSubjectPrefix+' '+ VersionTag+'=(.*) '+ RefInodeTag+'=(.*) '+ FsNameTag+'='+MagicStartDelim+'(.*)'+MagicEndDelim, msg['Subject']) subject = ( DirentSubjectPrefix+ " "+VersionTag+"="+GMAILFS_VERSION+ " "+RefInodeTag+"="+m.group(2)+ " "+FsNameTag+"="+MagicStartDelim+ fsNameVar +MagicEndDelim ) bodytmp = fixQuotedPrintable(msg.as_string()) m = re.search(FileNameTag+'='+FileStartDelim+'(.*)'+FileEndDelim+ ' '+PathNameTag+'='+PathStartDelim+'(.*)'+PathEndDelim+ ' '+LinkToTag+'='+LinkStartDelim+'(.*)'+LinkEndDelim, bodytmp) body = (FileNameTag+"="+FileStartDelim+ name +FileEndDelim+ " "+PathNameTag+"="+PathStartDelim+ fspath +PathEndDelim+ " "+LinkToTag+"="+LinkStartDelim+ m.group(3) +LinkEndDelim ) msg = mkmsg(username, fsNameVar, subject, body) msgid = imap_append("rename", self.imap, msg) if msgid > 0: log_debug("Sent subject '"+subject+"' ok") trashMessage(self.imap, msg) if self.inodeCache.has_key(path): #del self.inodeCache[path] # this cache flushing in unfortunate but currently necessary # to avoid problems with hard links losing track of # number of the number of links self.inodeCache = {} return 0 else: e = OSError("Couldnt send mesg"+path) e.errno = ENOSPC raise e #@-node:rename #@+node:link def link(self, path, path1): log_debug1("hard link: path='%s', path1='%s'" % (path, path1)) inode = self.getinode(path) if not (inode.mode & S_IFREG): e = OSError("hard links only supported for regular files not directories:"+path) e.errno = EPERM raise e inode.nlink+=1 inode.update() self._mkfileOrDir(path1,None,inode.mode,inode.ino,0,1) return 0 #@-node:link #@+node:chmod def chmod(self, path, mode): log_debug1("chmod called with path: '%s' mode: '%o'" % (path, mode)) inode = self.getinode(path) inode.mode = (inode.mode & ~(S_ISUID|S_ISGID|S_ISVTX|S_IRWXU|S_IRWXG|S_IRWXO)) | mode inode.update() return 0 #@-node:chmod #@+node:chown def chown(self, path, user, group): log_debug1("chown called with user:"+str(user)+" and group:"+str(group)) inode = self.getinode(path) inode.uid = user inode.gid = group inode.update() return 0 #@-node:chown #@+node:truncate def truncate(self, path, size): log_debug1("truncate "+path+" to size:"+str(size)) inode = self.getinode(path) # this is VERY lazy, we leave the truncated data around # it WILL be harvested when we grow the file again or # when we delete the file but should probably FIX inode.size = size; log_debug("truncate('%s') forcing update" % path) inode.update() return 0 #@-node:truncate #@+node:getxattr def getxattr(self, path, attr, size): log_debug1("getxattr('%s', '%s', '%s')" % (path, attr, size)) inode = self.getinode(path) # TODO check to make sure we don't overflow size if attr not in inode.xattr: return -ENODATA ret = inode.xattr[attr] if size == 0: return len(ret) return ret #@-node:getxattr #@+node:setxattr def setxattr(self, path, attr, value, dunno): log_debug1("setxattr('%s', '%s', '%s', '%s')" % (path, attr, value, dunno)) inode = self.getinode(path) inode.xattr[attr] = value inode.update() return 0 #@-node:setxattr #@+node:removexattr def removexattr(self, path, attr, value, dunno): log_debug1("removexattr('%s', '%s')" % (path, attr)) inode = self.getinode(path) try: del inode.xattr[attr] except: return -ENOATTR inode.update() return 0 #@-node:removexattr #@+node:listxattr def listxattr(self, path, size): log_debug1("listxattr('%s', '%s')" % (path, size)) inode = self.getinode(path) # We use the "user" namespace to please XFS utils attrs = [] for attr in inode.xattr: log_debug1("listxattr() attr: '%s'" % (attr)) attrs.append(attr) if size == 0: # We are asked for size of the attr list, ie. joint size of attrs # plus null separators. return len("".join(aa)) + len(aa) log_debug1("all attrs: (%s)" % (string.join(attrs, ", "))) return attrs #@-node:listxattr #@+node:mknod def mknod(self, path, mode, dev): """ Python has no os.mknod, so we can only do some things """ log_info("mknod('%s')" % (path)) if S_ISREG(mode) | S_ISFIFO(mode) | S_ISSOCK(mode): self._mkfileOrDir(path,None,mode,-1,0,1) #open(path, "w") else: return -EINVAL #@-node:mknod def _mkfileOrDir(self,path,path2,mode,inodenumber,size,nlinks): log_debug1("_mkfileOrDir('%s', '%s',...)" % (path, path2)) ind = string.rindex(path,'/') log_debug("ind:"+str(ind)) dirpath = path[:ind] if len(dirpath)==0: dirpath = "/" name = path[ind+1:] log_debug("dirpath:"+dirpath+" name:"+name) fspath = _pathSeparatorEncode(dirpath) if path2 == None: path2 = "" if inodenumber == -1: inodeno = int(time.time()) else: inodeno = inodenumber subject =( DirentSubjectPrefix+ " "+VersionTag+"="+GMAILFS_VERSION+ " "+RefInodeTag+"="+str(inodeno)+ " "+FsNameTag+"="+MagicStartDelim+ fsNameVar +MagicEndDelim+ " "+FileNameTag+"="+FileStartDelim+ name +FileEndDelim+ " "+PathNameTag+"="+PathStartDelim+ fspath +PathEndDelim+ "") body = (""+FileNameTag+"="+FileStartDelim+ name +FileEndDelim+ " "+PathNameTag+"="+PathStartDelim+ fspath +PathEndDelim+ " "+LinkToTag+"="+LinkStartDelim + path2 +LinkEndDelim+ "") dirent_msg = mkmsg(username, fsNameVar, subject, body) dirent_msgid = imap_append("mk dirent", self.imap, dirent_msg) if dirent_msgid > 0: log_debug("_mkfileOrDir() 1 Sent '"+subject+"' ok") else: e = OSError("Couldnt send mesg in _mkfileOrDir "+path) e.errno = ENOSPC raise e # only create inode if number not provided inode_msg = None if inodenumber == -1: timeString = str(int(time.time())) subject = ( InodeSubjectPrefix+ " "+VersionTag+"="+GMAILFS_VERSION+ " "+InodeTag+"="+str(inodeno)+ " "+DevTag+"=11 "+NumberLinksTag+"="+str(nlinks)+ " "+FsNameTag+"="+MagicStartDelim+ fsNameVar +MagicEndDelim+ "") body = ( ""+ModeTag+"="+str(mode)+ " "+UidTag+"="+str(os.getuid())+" "+GidTag+"="+str(os.getgid())+" "+SizeTag+"="+str(size)+ " "+AtimeTag+"="+timeString+ " "+MtimeTag+"="+timeString+ " "+CtimeTag+"="+timeString+ " "+BlockSizeTag+"="+str(DefaultBlockSize)+ "") inode_msg = mkmsg(username, fsNameVar, subject, body) inode_msgid = imap_append("mk inode", self.imap, inode_msg) if inode_msgid > 0: log_debug("_mkfileOrDir() 2 Sent '"+subject+"' ok") else: e = OSError("Couldnt send mesg"+path) e.errno = ENOSPC raise e # this is an optimization so that we don't have # to refetch the inode log_debug2("about to instantiate GmailInode() with msg subj: '%s'" % (dirent_msg['Subject'])) inode = GmailInode(dirent_msg, inode_msg, self.imap) self.inodeCache[path] = inode # end optimization log_debug1("done _mkfileOrDir(%s, %s,...)" % (path, path2)) #@+node:mkdir def mkdir(self, path, mode): log_debug1("mkdir path:"+path+" mode:"+str(mode)) self._mkfileOrDir(path,None,mode|S_IFDIR, -1,1,2) inode = self.getinode(path) ind = string.rindex(path,'/') log_debug("ind:"+str(ind)) parentdir = path[:ind] if len(parentdir)==0: parentdir = "/" parentdirinode = self.getinode(parentdir) parentdirinode.nlink+=1 parentdirinode.update() del self.inodeCache[parentdir] #@-node:mkdir #@+node:utime def utime(self, path, times): log_debug1("utime for path:"+path+" times:"+str(times)) inode = self.getinode(path) inode.atime = times[0] inode.mtime = times[1] return 0 #@-node:utime #@+node:open def open(self, path, flags): log_debug1("gmailfs.py:Gmailfs:open: %s" % path) try: inode = self.getinode(path) f = OpenGmailFile(inode) self.openfiles[path] = f return 0 except: _logException("Error opening file: "+path) e = OSError("Error opening file: "+path) e.errno = EINVAL raise e #@-node:open #@+node:read def read(self, path, readlen, offset): try: log_debug1("gmailfs.py:Gmailfs:read(len=%d, offset=%d, path='%s')" % (readlen, offset, path)) f = self.openfiles[path] buf = f.read(readlen,offset) arr = array.array('c') arr.fromlist(buf) rets = arr.tostring() return rets except: _logException("Error reading file"+path) e = OSError("Error reading file"+path) e.errno = EINVAL raise e #@-node:read #@+node:write def write(self, path, buf, off): try: log_debug2("gmailfs.py:Gmailfs:write: %s" % path) if log.isEnabledFor(logging.DEBUG): log_debug3("writing file contents: ->"+str(buf)+"<-") f = self.openfiles[path] written = f.write(buf,off) log_debug2("wrote %d bytes to file: '%s'" % (written, f)) return written except: _logException("Error opening file"+path) e = OSError("Error opening file"+path) e.errno = EINVAL raise e #@-node:write #@+node:release def release(self, path, flags): log_debug1("gmailfs.py:Gmailfs:release: %s %x" % (path, int(flags))) # I saw a KeyError get thrown out of this once. Looking back in # the logs, I saw two consecutive release: # 01/20/10 17:47:47 INFO gmailfs.py:Gmailfs:release: /linux-2.6.git/.Makefile.swp 32768 # 01/20/10 17:47:49 INFO gmailfs.py:Gmailfs:release: /linux-2.6.git/.Makefile.swp 32769 # f = self.openfiles[path] f.close() del self.openfiles[path] return 0 #@-node:release def get_quota_info(self): # not really interesting because we don't care how much # is in the entire account, just our particular folder #resp, data = self.imap.getquota("") #log_info("quota resp: '%s'/'%s'" % (resp, data)) # response looks like: # [['"linux_fs_3" ""'], ['"" (STORAGE 368 217307895)']] resp, data = self.imap.getquotaroot(fsNameVar) storage = data[1][0] m = re.match('"" \(STORAGE (.*) (.*)\)', storage) used_blocks = int(m.group(1)) allowed_blocks = int(m.group(2)) return [used_blocks * 1024, allowed_blocks * 1024] #@+node:statfs def statfs(self): log_info("statfs()") """ Should return a tuple with the following 6 elements: - blocksize - size of file blocks, in bytes - totalblocks - total number of blocks in the filesystem - freeblocks - number of free blocks - availblocks - number of blocks available to non-superuser - totalfiles - total number of file inodes - freefiles - nunber of free file inodes Feel free to set any of the above values to 0, which tells the kernel that the info is not available. """ st = fuse.StatVfs() block_size = 1024 quotaBytesUsed, quotaBytesTotal = self.get_quota_info() blocks = quotaBytesTotal / block_size quotaPercent = 100.0 * quotaBytesUsed / quotaBytesTotal blocks_free = (quotaBytesTotal - quotaBytesUsed) / block_size blocks_avail = blocks_free # I guess... log_debug("%s of %s used. (%s)\n" % (quotaBytesUsed, quotaBytesTotal, quotaPercent)) log_debug("Blocks: %s free, %s total\n" % (blocks_free, blocks)) files = 0 files_free = 0 namelen = 80 st.f_bsize = block_size st.f_frsize = block_size st.f_blocks = blocks st.f_bfree = blocks_free st.f_bavail = blocks_avail st.f_files = files st.f_ffree = files_free return st #@-node:statfs #@+node:fsync def fsync(self, path, isfsyncfile): log_debug1("gmailfs.py:Gmailfs:fsync: path=%s, isfsyncfile=%s" % (path, isfsyncfile)) inode = self.getinode(path) f = self.openfiles[path] f.commitToGmail() return 0 #@-node:fsync def fetch_msg_for_path(self, path): try: log_debug2("check getnodemsg path: '"+path+"'") ind = string.rindex(path,'/') log_debug2("ind:"+str(ind)) dirpath = path[:ind] if len(dirpath)==0: dirpath = "/" name = path[ind+1:] log_debug("dirpath: "+dirpath+" name:"+name) fspath = _pathSeparatorEncode(dirpath) q = []; q.append(str(FileNameTag+'='+FileStartDelim + name + FileEndDelim)); q.append(str(PathNameTag+'='+PathStartDelim + fspath + PathEndDelim)); msg = getSingleMessageByQuery(self.imap, q) return msg except: _logException("no slash in path: '%s'" % path) return None def getinode(self, path): if self.inodeCache.has_key(path): log_debug2("getinode() cache hit for '%s'" % (path)) return self.inodeCache[path] else: log_debug2("getinode() cache miss for '%s'" % (path)) dirent_msg = self.fetch_msg_for_path(path) if dirent_msg == None: return None subject = dirent_msg['Subject'] m = re.match(DirentSubjectPrefix+' '+ VersionTag+'=(.*) '+ RefInodeTag+'=(.*) '+ FsNameTag+'='+MagicStartDelim+'(.*)'+MagicEndDelim, subject.replace('\u003d','=')) inodeNumber = m.group(2) inode_msg = getSingleMessageByQuery(self.imap, [InodeTag+'='+inodeNumber+'']) inode = GmailInode(dirent_msg, inode_msg, self.imap) if inode: self.inodeCache[path] = inode return inode #@-others #@-node:class Gmailfs #@+node:mainline # Setup logging log = logging.getLogger('gmailfs') #defaultLogLevel = logging.WARNING defaultLogLevel = logging.DEBUG log.setLevel(defaultLogLevel) defaultLogFormatter = logging.Formatter("%(asctime)s %(levelname)-10s %(message)s", "%x %X") # log to stdout while parsing the config while defaultLoggingHandler = logging.StreamHandler(sys.stdout) _addLoggingHandlerHelper(defaultLoggingHandler) GmailConfig([SystemConfigFile,UserConfigFile]) try: libgmail.ConfigLogs(log) except: pass def main(mountpoint, namedOptions): server = Gmailfs(namedOptions,mountpoint,version="gmailfs 0.8.0",usage='',dash_s_do='setsingle') server.parser.mountpoint = mountpoint server.parse(errex=1) server.flags = 0 server.multithreaded = False; server.main() if __name__ == '__main__': main(1, "2") #@-node:mainline #@-others #@-node:@file gmailfs.py #@-leo