Logo Search packages:      
Sourcecode: sbackup version File versions  Download package

sbackupd.py

#!/usr/bin/python
#
# Simple Backup suit
# 
# Running this command will execute a single backup run according to a configuration file
#
# Author: Aigars Mahinovs <aigarius@debian.org>
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

import sys
import os
import errno
import atexit
import stat
import datetime
import time
import os.path
import cPickle as pickle
import shutil
import ConfigParser
import re
import socket
import tempfile
import upgrade_backups
import getopt
import grp
from gettext import gettext as _

try:
    import gnomevfs
except ImportError:
    import gnome.vfs as gnomevfs

# Classes we use but that are not worth putting them in their own module
class MyConfigParser(ConfigParser.ConfigParser):
       def __init__(self, verbose = False):
               self.verbose = verbose
               if self.verbose: print "MyConfigParser.__init__"
               ConfigParser.ConfigParser.__init__(self);
               self.valid_options = {}
               self.filename_from_argv = None
               self.filename = ""
               self.argv_options = {}

       def set_valid_options(self, valid_options, parse_commandline = False):
               pass
               self.valid_options = valid_options
               if parse_commandline and self.valid_options:
                       self.parse_commandline()

       def read(self, filename):
               if self.verbose: print "MyConfigParser.read(%s)" % filename

               self.filename = self.filename_from_argv or filename
               retValue = ConfigParser.ConfigParser.read(self, self.filename)
               if self.valid_options: self.validate_config_file_options()
               if self.argv_options: self.validate_argv_options()
               if self.valid_options: self.validate_option_values()
               return retValue

       def optionxform(self, option):
            return str( option )

       def parse_commandline(self):
               if self.verbose: print "MyConfigParser.parse_commandline"
               argv_parameters = [ "config-file=" ]
               for section, data in self.valid_options.iteritems():
                       for (key, vtype) in data.iteritems():
                               if (key == '*') : continue
                               argv_parameters.append("%s:%s="% (section, key))
               (options, reminder) = \
                       getopt.getopt(sys.argv[1:], '', argv_parameters)
               self.argv_options = dict(options)
               if (self.argv_options.has_key("--config-file")):
                       self.filename_from_argv = self.argv_options["--config-file"]
                       del self.argv_options["--config-file"]


       def validate_config_file_options(self):
               if self.verbose: print "MyConfigParser.validate_config_file_options"
               if (self.valid_options is None): return
               for section in self.sections():
                       try:
                               for key in self.options(section):
                                       if (not self.valid_options.has_key(section)):
                                               raise Exception ("section [%s] in %s should not exist, aborting" % (section, config))
                                       if (self.valid_options[section].has_key(key) or
                                               self.valid_options[section].has_key('*')):
                                               continue
                                       raise Exception ("key %s in section %s in file %s is not known,\na typo possibly?"
                                               % (key, section, config))
                       except Exception, e:
                               print str(e)
                               sys.exit(1);

       def validate_argv_options(self):
               if self.verbose: print "MyConfigParser.validate_argv_options"
               for parameter, value in self.argv_options.iteritems():
                       parameter = parameter[2:]
                       (section, key) = parameter.split(":")
                       self.set(section,key, value)

       def validate_option_values(self):
               # check if all keys defined in valid_options have values now
             pass
#               for section, data in self.valid_options.iteritems():
#                       for key, vtype in data.iteritems():
#                               if key == '*': continue
#                               try:
#                                       value = self.get(section, key)
#                               except Exception, e:
#                                       raise Exception ("The definition for %s:%s is missing.\nDefine in the config file or on the command line" %
#                                               (section, key))
#                               if vtype is list:
#                                       pass
#                               else:
#                                       self.set(section, key, vtype(value))

       def all_options(self):
               retVal = []
               for section in self.sections():
                       for key in self.options(section):
                               value = self.get(section, key, raw = True)
                               retVal.append( (key,value))
               return retVal

       def __str__(self):
               retVal = []
               for section, sec_data in self._sections.iteritems():
                       retVal.append("[%s]" % section)
                       [retVal.append("%s = %s" % (o, repr(v)))
                               for o, v in sec_data.items()
                               if o != '__name__']
               return "\n".join(retVal)


# Default values, constants and the like
our_options = {
 'general' : { 'target' : str , 'lockfile' : str , 'maxincrement' : int , 'format' : int, 'purge' : str },
 'dirconfig' : { '*' : str },
 'exclude' : { 'regex' : list, 'maxsize' : int },
 'places' : { 'prefix' : str }
}

# Define default values & load config file

config = "/etc/sbackup.conf"
target = "/var/backup/"
increment = 0
lockfile = "/var/lock/sbackup.lock"
hostname = socket.gethostname()
maxincrement = 7
# Backup format: 1 - allways use .tar.gz
format=1
os.umask( 027 )

# directories allready added to the archive
dirs_in = {"/" : 1}

dirconfig = { "/etc/":1, "/home/":1, "/usr/local/": 1, "/root/":1, "/var/": 1, "/var/cache/":0, "/var/spool/":0, "/var/tmp/":0 }
gexclude = [r"\.mp3",r"\.avi",r"\.mpeg",r"\.mkv",r"\.ogg", r"\.iso"]
maxsize = 10*1024*1024

# Check our user id
if os.geteuid() != 0: sys.exit (_("Currently backup is only runnable by root"))

# [Bug 112540] : Let the admin group have read access to the backup dirs
try :
      # The uid is still root and the gid is admin
      os.setgid( grp.getgrnam("admin").gr_gid )
except Exception, e: 
      print "W: Failed to set the gid to 'admin' one :" + str(e)
      pass

# second we read the config file, so must check if the user provided one on the
#   commando line.

#try:
conf = MyConfigParser()
conf.set_valid_options( our_options, parse_commandline = True)
conf.read( config )
#except Exception, e:
#       print "Error while reading config,\n"+str(e)
#       sys.exit(1)
#else:
#       pass

for option, value in conf.all_options():
       # skip options with a funny name, notably the dirs in [dirconfig]
       if re.search ("\W", option): continue
       # print "setting %s = %s" % (option, value)
       globals()[option]=value

if not target:
    print _("Option target is missing, aborting.")
    sys.exit(1)    

if conf.has_section( "dirconfig" ):
    dirconfig = dict([ (k, int(v)) for k,v in conf.items("dirconfig") ] )
if conf.has_option( "exclude", "regex" ):
    gexclude = str(conf.get( "exclude", "regex" )).split(",")

rexclude = [ re.compile(p) for p in gexclude if len(p)>0]

flist = False
flistid = 0
flist_name = ""
fprops = False
fullsize = 0L

def btree_r_add( adir ):
      """Add a directory to the btree with reversed recursion - take defaults from parent"""
      global btree
      
      os.path.normpath( adir )
      
      parent2 = os.path.split( adir )[0]
      parent = parent2
      if parent == "/":
            parent = ""
      if adir in btree:
            pass
      elif parent in btree:
            props = btree[parent]
            for child in os.listdir( parent2 ):
                  btree[os.path.normpath(parent+"/"+child)] = props
            btree[parent] = (-1, btree[parent][1], btree[parent][2])
      else:
            btree_r_add( parent )
            props = btree[parent]
            for child in os.listdir( parent2 ):
                  btree[os.path.normpath(parent+"/"+child)] = props
            btree[parent] = (-1, btree[parent][1], btree[parent][2])

def is_parent( parent, child ):
      """ Compares directories - returns child only if it is a child of the parent """
      if str(child)[0:len(parent)] == parent:
            return child
      else:
            return False

def do_backup_init( ):
      global flist, flistid, flist_name, fprops, fpropsid, fprops_name
      if local:
            (flistid, flist_name) = tempfile.mkstemp()
            flist = os.fdopen( flistid, "w" )
            fprops = open(tdir+"/fprops", "w")
      else:
            (flistid, flist_name) = tempfile.mkstemp()
            flist = os.fdopen( flistid, "w" )
            (fpropsid, fprops_name) = tempfile.mkstemp()
            fprops = os.fdopen( fpropsid, "w" )

def do_backup_finish( ):
      flist.close()
      fprops.close()
      tarline = "tar -czS -C / --no-recursion --ignore-failed-read --null -T "+flist_name+" "
      if local:
            tarline = tarline+" --force-local -f "+tdir.replace(" ", "\ ")+"/files.tgz"
            tarline = tarline+" 2>/dev/null"
            os.system( tarline )
            shutil.move( flist_name, tdir+"/flist" )
            os.chmod( tdir+"/flist", 0640 )
      else:
            tarline = tarline+" 2>/dev/null"
            turi = gnomevfs.URI( tdir+"/files.tgz" )
            tardst = gnomevfs.create( turi, 2 )
            tarsrc = os.popen( tarline )
            shutil.copyfileobj( tarsrc, tardst, 100*1024 )
            tarsrc.close()
            tardst.close()
            s1 = open( fprops_name, "r" )
            turi = gnomevfs.URI( tdir+"/fprops" )
            d1 = gnomevfs.create( turi, 2 )
            shutil.copyfileobj( s1, d1 )
            s1.close()
            d1.close()
            s2 = open( flist_name, "r" )
            turi = gnomevfs.URI( tdir+"/flist" )
            d2 = gnomevfs.create( turi, 2 )
            shutil.copyfileobj( s2, d2 )
            s2.close()
            d2.close()

def do_add_dir ( dirname, props ):
      do_add_file( dirname, props )

def do_add_file( dirname, props ):
      global fullsize
      parent = os.path.split( dirname )[0]
      if not dirs_in.has_key(parent) and parent != dirname:
            s = os.lstat( parent )
            do_add_dir( parent, str(s.st_mode)+str(s.st_uid)+str(s.st_gid)+str(s.st_size)+str(s.st_mtime))
            fullsize += s.st_size
      flist.write( dirname+"\000" )
      fprops.write( props+"\000" )
      dirs_in[ dirname ] = 1

def do_backup( adir ):
      """ Finds all files to be backuped in the directory and calls respective backup suroutines """
      s = os.lstat(adir)
      parent = os.path.split( adir )[0]
      if not os.path.isdir(adir) or os.path.islink(adir):
            if s.st_size > maxsize or ( prev.has_key( adir ) and prev[adir]== str(s.st_mode)+str(s.st_uid)+str(s.st_gid)+str(s.st_size)+str(s.st_mtime) ):
                return []
            for r in rexclude:
                if r.search( adir ):
                  return []
            do_add_file( adir, str(s.st_mode)+str(s.st_uid)+str(s.st_gid)+str(s.st_size)+str(s.st_mtime) )
      else:
            if not increment:
                  do_add_dir( adir, str(s.st_mode)+str(s.st_uid)+str(s.st_gid)+str(s.st_size)+str(s.st_mtime) )
            for child in os.listdir( adir ):
                  if os.path.isdir( adir+"/"+child ) and not os.path.islink( adir+"/"+child ):
                      do_backup( adir+"/"+child )
                  else:
                      try: s = os.lstat( adir+"/"+child )
                      except: continue
                      if maxsize > 0 and s.st_size > maxsize:
                        continue
                      if ( prev.has_key( adir+"/"+child ) and prev[adir+"/"+child]== str(s.st_mode)+str(s.st_uid)+str(s.st_gid)+str(s.st_size)+str(s.st_mtime) ): 
                        continue
                      skip=False
                      for r in rexclude:
                        if r.search( adir+"/"+child ):
                            skip=True
                      if skip:
                        continue
                      do_add_file( adir+"/"+child, str(s.st_mode)+str(s.st_uid)+str(s.st_gid)+str(s.st_size)+str(s.st_mtime) )

# End of helpfull functions :)
            
good = False

# Create the lockfile so noone disturbs us
try: 
      fd_lck = open( lockfile, "r" )
except IOError: # means the lockfile doesn't exists
      good = True 
if not good :
      # the lockfile exists, is it valid ?
      last_sb_pid = fd_lck.read()
      if (last_sb_pid and os.path.lexists("/proc/"+last_sb_pid) and "sbackupd" in str(open("/proc/"+last_sb_pid+"/cmdline").read()) ) :
            sys.exit (_("E: Another Simple Backup daemon already running: exiting"))
      else :
            fd_lck.close()
            os.remove (lockfile)
            good = True

try:
      lock = open( lockfile, "w+" )
      lock.write(str(os.getpid()))
      lock.close()
except IOError:
      print _("E: Can't create a lockfile: "), sys.exc_info()[1]
      sys.exit(1)

def exitfunc():
      # All done
      # Remove lockfile
      os.remove (lockfile)

atexit.register(exitfunc)

# Checking if the target directory is local or remote
local = True

try:
    if gnomevfs.get_uri_scheme( target ) == "file":
      target = gnomevfs.get_local_path_from_uri( target )
    else:
      local = False
except:
    pass

# Checking if the target directory exists (or can be created)
ok = -1
tinfo = False
try:    # Get target directory info
        tinfo = gnomevfs.get_file_info( target )
except:
        try:    # Try to create it, in case, it doesn't exist
                gnomevfs.make_directory( target, 0750 )
                tinfo = gnomevfs.get_file_info( target )
        except:
                ok = False
try:
    if tinfo.valid_fields == 0:
        ok = False
except:
    ok = False

if ok==-1:
        # Now try to write to the target dir
        try:
                test = str( time.time() )
                gnomevfs.make_directory( target+"/"+test, 0700 )
                gnomevfs.remove_directory( target+"/"+test )
                ok = True
        except:
                ok = False

if not ok:
      print _("E: Target directory is not writable - please test in simple-config-gnome!")
      sys.exit(1)
                                                                                                                                                                                                                                                                                                                                                                                                                                                
# Upgrade directories to new format
# and purge old backups

purge = 0
if conf.has_option("general", "purge"):
    purge = conf.get("general", "purge")

upgrader = upgrade_backups.SBUpgrade()
upgrader.upgrade_target( target, purge )

# Determine whether to do a full or incremental backup

r = re.compile(r"^(\d{4})-(\d{2})-(\d{2})_(\d{2})[\:\.](\d{2})[\:\.](\d{2})\.\d+\..*?\.(.+)$")

if local:
    listing = os.listdir( target )
    listing = filter( r.search, listing )
else:
    d = gnomevfs.open_directory( target )
    listing = []
    for f in d:
      if f.type == 2 and f.name != "." and f.name != ".." and r.search( f.name ):
          listing.append( f.name )

listing.sort()
listing.reverse()

# Check if these directories are complete and remove from the list those that are not
for adir in listing[:]:
      if local and not os.access( target+"/"+adir+"/ver", os.F_OK ):
            listing.remove( adir )
      if not local and not gnomevfs.exists( target+"/"+adir+"/ver" ):
            listing.remove( adir ) #TODO - check more stuff

prev = {}
base = False
maxincrement = int(maxincrement)

if listing == []:
      increment = 0     # No backups found -> make a full backup
else:
      m = r.search( listing[0] )
      if m.group( 7 ) == "ful":  # Last backup was full backup
            if (datetime.date.today() - datetime.date(int(m.group(1)),int(m.group(2)),int(m.group(3)) ) ).days <= maxincrement :
                # Less then maxincrement days passed since that -> make an increment
                  increment = time.mktime((int(m.group(1)),int(m.group(2)),int(m.group(3)),int(m.group(4)),int(m.group(5)),int(m.group(6)),0,0,-1))
                  base = listing[0]
                  try:
                        prev = dict( zip(str( gnomevfs.read_entire_file( target+"/"+base+"/flist" ) ).split( "\000" ), str( gnomevfs.read_entire_file( target+"/"+base+"/fprops" )).split( "\000" )) )
                  except:
                        increment = 0  # Last backup is somehow damaged 
            else:
                  increment = 0      # Too old -> make full backup
      else: # Last backup was an increment - lets search for the last full one
            r2 = re.compile(r"ful$")
            for i in listing:
                  try: 
                        for a,b in zip(str( gnomevfs.read_entire_file( target+"/"+i+"/flist" ) ).split( "\000" ), str( gnomevfs.read_entire_file( target+"/"+i+"/fprops" )).split( "\000" )) :
                              if not prev.has_key(a) : # We always keep the newer incremental file info
                                    prev[a]=b
                  except:
                        prev = {}          # One of the incremental backups is bad -> make a new full one
                        increment = 0
                        break
                  if r2.search( i ):
                        m = r.search( i )
                        if (datetime.date.today() - datetime.date(int(m.group(1)),int(m.group(2)),int(m.group(3)) ) ).days <= maxincrement :
                              # Last full backup is fresh -> make an increment
                              m = r.search( listing[0] )
                              increment = time.mktime((int(m.group(1)),int(m.group(2)),int(m.group(3)),int(m.group(4)),int(m.group(5)),int(m.group(6)),0,0,-1))
                              base = listing[0]
                        else: # Last full backup is old -> make a full backup
                            increment = 0
                            prev = {}
                        break
            else:
                  increment = 0            # No full backup found 8) -> lets make a full backup to be safe
                  prev = {}


# Determine and create backup target directory

tdir = target + "/" + datetime.datetime.now().isoformat("_").replace( ":", "." ) + "." + hostname + "."
if increment != 0:
      tdir = tdir + "inc/"
else:
      tdir = tdir + "ful/"

if local:
    os.makedirs( tdir, 0750 )
else:
    gnomevfs.make_directory( tdir, 0750 )

# Create '.../base' here, if incremental backup

if base:
    if local:
      f = open( tdir+"base", 'w' )
    else:
      f = gnomevfs.create( tdir+"base", 2 )
    f.write( base+"\n" )
    f.close

tar = True

# Reduce the priority, so not to interfere with other processes
os.nice(20)

# Initiate backup tree structure

defexcludes = ["", "/dev", "/proc", "/sys", "/tmp"]
btree = {}
for defexclude in defexcludes:
    if dirconfig.has_key(defexclude+"/"):
            btree[defexclude] = (dirconfig.pop(defexclude+"/"),0,[])
    else:
            btree[defexclude] = (0,0,[])

# Populate the backup tree structure
sdirs = dirconfig.keys()
sdirs.sort()
for adir in sdirs:
      if os.path.exists( adir ):
          btree_r_add( adir )
          btree.update( btree.fromkeys( [adir2 for adir2 in btree.keys() if is_parent(adir, adir2)] , (dirconfig[adir],0,[]) ) )
#           btree[os.path.normpath(adir)] = (dirconfig[adir],0,[])

# Remove target from the backup
if local:
      btree_r_add( target )
      btree.update( btree.fromkeys( [adir2 for adir2 in btree.keys() if is_parent(target, adir2)] , (0,0,[]) ) )
      btree[os.path.normpath(target)] = (0,0,[])

# Write excludes
if local:
    pickle.dump( gexclude, open(tdir+"excludes","w") )
else:
    pickle.dump( gexclude, gnomevfs.create(tdir+"excludes", 2) )

# Backup list of installed packages (Debian only part)
try:
    command = "dpkg --get-selections"
    s = os.popen( command )
    if local:
        d = open( tdir+"packages", "w" )
    else:
        d = gnomevfs.create( tdir+"packages", 2 )
    shutil.copyfileobj( s, d )
    s.close()
except:
    pass
# End of Debian only part

# Make the backup ...
do_backup_init()
bdirs = btree.keys()
bdirs.sort()

for adir in bdirs:
      if adir == "":
            adir2 = "/"
      else:
            adir2 = adir
      if btree[adir][0] == 1 and os.path.exists( adir2 ):
            do_backup( adir2 )

# Find out the free space in target and abort the backup if there is not enough space
if local:
    vstat = os.statvfs( tdir )
    if (vstat.f_bavail * vstat.f_bsize) <= fullsize:
      print _("E: Not enough free space on the target directory for the planned backup")
      sys.exit( 1 )

do_backup_finish()
# ... done.

if local:
    f = open( tdir+"ver", 'w' )
else:
    f = gnomevfs.create( tdir+"ver", 2 )

f.write( "1.4\n" )
f.close()



# Write statistics
# TODO for next versions #

sys.exit( 0 )


Generated by  Doxygen 1.6.0   Back to index