Logo Search packages:      
Sourcecode: upstreamdev version File versions  Download package

moduleloader.py

#!/usr/bin/python
#
# Upstream - log file aggregator and report tool for *nix systems.
# Copyright (C) 2006  Ryan Zeigler (zeiglerr@gmail.com)
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

# This module provides support for our module import facilities
# there seems to exist a more "pythony" way of doing this via
# import hooks.  This is an area of future development, i.e. determine
# if it is viable for what we want to do.  Currently we use unbound
# import hooks which means we never have to deal with complex implementation
# details


# TODO some of the __repr__ seem to try to concatenate strings with non-strings

import glob, imp, sys, os, threading

#  This module was built with the goal of maximum fault tolerance for
#  any imported modules.  If there is a discovered case in which
#  fault tolerance

MLOAD_NOT_LIST = 0
MLOAD_EMPTY_LIST = 1
MLOAD_HAS_NONSTR = 2

DEBUG_NONE = 0
DEBUG_ALL = 1

class ModuleLoaderInitException(Exception):
      def __init__(self, err_type):
            Exception.__init__(self)
            self.err_type = err_type
      def __repr__(self):
            return "raised ModuleLoaderInitException(" + self.err_type + ")"
      def __str__(self):
            if self.err_type == MLOAD_NOT_LIST:
                  return "Error: Package list was not a list"
            elif self.err_type == MLOAD_EMPTY_LIST:
                  return "Error: Package list was empty"
            elif self.err_type == MLOAD_HAS_NONSTR:
                  return "Error: Package list contained non-strings"
            else:
                  return "Error: unknown?"
                  
                  
class IncorrectModuleReturnType(Exception):
      def __init__(self, found_type, expected_type):
            Exception.__init__(self)
            self.found_type = found_type
            self.expected_type = expected_type
      def __repr__(self):
            return "raised IncorrectModuleReturnType(" + self.found_type + "," + self.expected_type + ")"
      def __str__(self):
            return "Found type: " + self.found_type + " Expected type: " + self.expected_type

# LoadedModule now extends from Thread type
class LoadedModule(threading.Thread):
      fault_tolerance = True
      debug_output = DEBUG_NONE
      # This expects the module fields to already exist
      # Note: any new module wrappers must have this exact argument list
      def __init__(self, module, fault_tolerance, debug_output):
            threading.Thread.__init__(self)
            self.fault_tolerance = fault_tolerance
            self.debug_output = debug_output
            self.module = module
            self.module_name = self.module.module_name
            self.module_description = self.module.module_description
      def __repr__(self):
            return "< loaded module with name : %s >" % self.module_name

# ModuleLoader now extends from Thread type
class ModuleLoader(threading.Thread):
      # Necessary attributes for a generic module
      # Subclasses may override this to provide for different required attributes
      necessary_attributes = ["module_name", "module_description"]
      necessary_attr_types = [str, str]
      load_stats = -1
      validation_status = -1
      # New classes should override the ModuleWrapper item
      ModuleWrapper = LoadedModule
      def __init__(self, pack_list, fault_tolerance=True, debug_output=DEBUG_NONE, use_threading = False):
            # Chain up
            threading.Thread.__init__(self)
            self.threaded = use_threading
            self.pack_list = pack_list
            self.debug_output = debug_output
            self.fault_tolerance = fault_tolerance
            self.valid_modules = []
            if use_threading:
                  self.start()
            else:
                  self.execute_load()
      
      def __repr__(self):
            return "ModuleLoader(" + str(self.pack_list) + ", " + str(self.fault_tolerance) + ", " + str(self.debug_output) + ")"
      
      def __getitem__(self, modid):
            if type(modid) is not str and type(modid) is not int:
                  raise TypeError
            # Find at index
            if type(modid) is int:
                  if modid >= len(self.valid_modules) or modid < 0:
                        raise IndexError
                  else:
                        return self.valid_modules[modid]
            # Find at id
            if type(modid) is str:        
                  for mod in self.valid_modules:
                        if mod.module_name == modid:
                              return mod
                  raise KeyError
            
      def __delitem__(self, modid):
            if type(modid) is not str and type(modid) is not int:
                  raise TypeError
            # Find at index
            if type(modid) is int:
                  if modid >= len(self.valid_modules) or modid < 0:
                        raise IndexError
                  else:
                        self.valid_modules.remove(self.valid_modules[modid])
            # Find at id
            if type(modid) is str:        
                  # This will already raise an exception if necessary
                  mod = self.__getitem__(modid)
                  self.valid_modules.remove(mod)
                  
                              
      def __len__(self):
            return len(self.valid_modules)
                  
      def __iter__(self):
            return ModuleLoaderIterator(self)
      # Methods utilized by the thread class
      def run(self):
            print "Running threaded load"
            self.execute_load()
            
      def execute_load(self):
            # Perform validation to ensure that we didn't end up invalid
            # parameters
            if type(self.pack_list) is not list:
                  raise ModuleLoaderInitException(MLOAD_NOT_LIST)
                  
            # Only perform an actual import if we have a non-zero list
            if len(self.pack_list) is not 0:
                  # Perform validation of the content of the pack_list                          
                  for p in self.pack_list:
                        if type(p) is not str:
                        # Prune from list if not a string
                              if self.fault_tolerance:
                                    self.pack_list.remove(p)
                              else:
                                    raise ModuleLoaderInitException(MLOAD_HAS_NONSTR)
                              
                  # Find all packages and import their modules
                  loaded_modules = []
                  for package_name in self.pack_list:
                        if self.debug_output >= DEBUG_ALL:
                              print "Importing %s" % package_name
                        imp_pack = __import__(package_name)
                        for plugin_name in imp_pack.__all__:
                              __import__(package_name + "." + plugin_name)
                              loaded_modules.append(getattr(imp_pack, plugin_name))
                              
                        #candidate_modules = []
                        #candidate_modules = candidate_modules + self.scan_directory(path_name)

                  # Load modules
                  #self.load_status = 0.0
                  #loaded_modules = []
                  #counter = 0
                  #for mod in candidate_modules:
                  #     tmp_module = self.load_module(mod)
                  #     if tmp_module:
                  #           loaded_modules.append(tmp_module)
                  #     counter = counter + 1
                  #     self.load_status = (counter + 0.0)/len(candidate_modules) 

                  # Validate modules
                  self.validation_status = 0.0
                  counter = 0
                  for mod in loaded_modules:
                        if self.validate_module(mod):
                              if self.debug_output >= DEBUG_ALL:
                                    print " Adding %s to valid modules" % (mod)
                              self.valid_modules.append(self.ModuleWrapper(mod, self.fault_tolerance, self.debug_output))
                        elif self.debug_output >= DEBUG_ALL:
                                    print " Not adding %s to valid modules" % (mod)
                        counter = counter + 1
                        self.validation_status = (counter + 0.0)/len(loaded_modules)
                                                
            else:
                  if not self.fault_tolerance:
                        raise ModuleLoaderInitException(MLOAD_EMPTY_LIST)
                        

      # Helper function.  This function scans a directory and returns
      # all of the *.py things it globs.  It returns a whole bunch of
      # truncated module names
      def scan_directory(self, path):
            if self.debug_output >= DEBUG_ALL:
                  print "Scanning directory: %s" % path
            if path[len(path) - 1] == '/':
                  py_pattern = path + "*.py"
                  pyc_pattern = path = "*.pyc"
                  pyo_pattern = path = "*.pyo"
            else:
                  py_pattern = path + "/*.py"
                  pyc_pattern = path + "/*.pyc"
                  pyo_pattern = path + "/*.pyo"
                  
            # Get globs
            found_py = glob.glob(py_pattern)
            found_pyc = glob.glob(pyc_pattern)
            found_pyo = glob.glob(pyo_pattern)
            
            # Convert to basename
            base_py = [os.path.basename(mod) for mod in found_py]
            base_pyc = [os.path.basename(mod) for mod in found_pyc]
            base_pyo = [os.path.basename(mod) for mod in found_pyo]
            
            stripped_py = [mod[0:mod.rfind(".py")] for mod in base_py]
            stripped_pyc = [mod[0:mod.rfind(".pyc")] for mod in base_pyc]
            stripped_pyo = [mod[0:mod.rfind(".pyo")] for mod in base_pyo]
            
            
            # Remove the remaining path names from the module, since 
            # We also have to add 1 to the index of the /
            found_modules = stripped_py
            for mod in stripped_pyc:
                  if mod not in found_modules:
                        if self.debug_output >= DEBUG_ALL:
                              print "Module %s does not have a corresponding source file" % mod
                        found_modules.append(mod)
                  elif self.debug_output >= DEBUG_ALL:
                        print "Module %s has a bytecode file" % mod
                        
            for mod in stripped_pyo:
                  if mod not in found_modules:
                        if self.debug_output >= DEBUG_ALL:
                              print "Module %s does not have a corresponding source file or standard bytecode file" % mod
                        found_modules.append(mod)
                  elif self.debug_output >= DEBUG_ALL:
                        print "Module %s has a bytecode file" % mode
            # Return the found modules
            return found_modules
            
      def load_module(self, modname):
            if self.debug_output >= DEBUG_ALL:
                  print "Attempting to 'find' module: %s" % modname
            file_handle, filename, description = imp.find_module(modname)
            loaded_module = None
            if not file_handle:
                  if self.debug_output >= DEBUG_ALL:
                        print "Failed 'finding' module (programming error or possible collision with builtin module): %s" % stripped_modname
            else:
                  try:
                        loaded_module = imp.load_module(modname, file_handle, modname, description)
                  except:
                        # Close our file handle
                        file_handle.close()
                        # If we are not using fault tolerance, reraise
                        if self.debug_output >= DEBUG_ALL:
                              print "Load failed on module: %s, attempting recovery" % modname
                              print sys.exc_info()[0]
                        if not self.fault_tolerance:
                              raise

                  file_handle.close()
            return loaded_module
            
      
      # Provide a string method
      def __str__(self):
            return "Module loader:\n" + repr(self.valid_modules   )
            
      # This is the bare minimum necessary for one of our         
      def validate_module(self, module):
            if self.debug_output >= DEBUG_ALL:
                  print "Validating module: %s" % module.__name__
            valid_fields = self.validate_fields(module) 
            valid_additional = self.validate_additional(module)
            return valid_fields and valid_additional
      # Determine if the module has the necessary fields to be a valid module
      # Subclasses should probably not have to override this method, and
      # instead, they should rely on overriding the "necessary_attributes" field
      def validate_fields(self, module):
            for field in self.necessary_attributes:
                  if self.debug_output >= DEBUG_ALL:
                        print " Validating fields %s : %s" % (field, hasattr(module, field))
                  if not hasattr(module, field):
                        return False
                  ind = self.necessary_attributes.index(field)
                  # This only runs when we have actually specified out to that
                  # type
                  if ind < self.necessary_attr_types:
                        # Ahh ye olde debugging output, you take up so much space
                        if self.debug_output >= DEBUG_ALL:
                              print " Validating field %s as type %s : %s" % (field, self.necessary_attr_types[ind], type(module.__dict__[field]) == self.necessary_attr_types[ind] and self.necessary_attr_types[ind] is not None)
                        
                        if not type(module.__dict__[field]) == self.necessary_attr_types[ind] and self.necessary_attr_types[ind] is not None:
                              return False
            # If we get to the end, we were successful
            return True
      # Determine if the module has the necessary activation hooks to be
      # a module.  Subclasses will probably have to reimplement this method
      # from scratch, since a default module provides no hooks into the module
      # and simply returns true.  DEBUG output can be retrieved by simply chaining
      # up, since there is no
      def validate_additional(self, module):
            return True
            
      # This is not actually used by default, but is provided as a convenience,
      # so that base classes do not have to reimplement it
      def validate_execution_hook(self, module, name, num_args):
            if self.debug_output >= DEBUG_ALL:
                  print " Module %s has attribute %s: %s " % (module, name, hasattr(module, name))
                  print " Module attribute %s is of type 'func_code': %s" % (name, hasattr(module.__dict__[name], "func_code"))
                  print " Module function %s has %d args: %s" % (name, num_args, module.__dict__[name].func_code.co_argcount is num_args)
            hasfunc = hasattr(module, name) 
            func_is_func = hasattr(module.__dict__[name], "func_code")
            func_has_correct_param = module.__dict__[name].func_code.co_argcount is num_args
            # Ryan: Is this ok? 
            # Fixed, good spot
            return hasfunc and func_is_func and func_has_correct_param
      
      # Deprecated: Use mappings instead  
      def module(self, mod_name):   
            if self.debug_output >= DEBUG_ALL:
                  print "Searching for module: %s" % mod_name     
            for x in self.valid_modules:
                  if self.debug_output >= DEBUG_ALL:
                        print "Looking at module: %s trying to find: %s" % (x, mod_name)
                  if x.module_name == mod_name or x.module.__name__ == mod_name:
                        return x
            return None


class ModuleLoaderIterator:
      def __init__(self, parent):
            self.parent = parent
            self.ind = -1
            
      def next(self):
            self.ind = self.ind + 1
            if self.ind == len(self.parent.valid_modules):
                  raise StopIteration
            else:
                  return self.parent.valid_modules[self.ind]


########################################################
# All classes/functions below this marker are deprecated,
# and will be removed soon
########################################################
class ModuleDirectoryScanner:
      def __init__(self, path, fault_tolerance, debug_output):
            self.duplicate_path =  False
            self.fault_tolerance = fault_tolerance
            self.debug_output = debug_output
            
            self.path = path
            
            if self.path not in sys.path:
                  sys.path.append(self.path)
                  if self.debug_output >= DEBUG_ALL:
                        print "Adding %s to path list" % self.path
                        print sys.path
                        
                  
            else:
                  if self.debug_output >= DEBUG_ALL:
                        print "Not adding %s to path list" % self.path
                        print sys.path
                  self.duplicate_path = True
            
            self.dir_modules = []
            if debug_output >= DEBUG_ALL:
                  print "Scanning directory: %s" % self.path
            self.scan()
            self.load()
                  
                  
      def __iter__(self):
            return ModuleDirectoryScannerIterator(self)           
      
      # Scan for possible modules that can be loaded  
      #def scan(self):
            #if self.path[len(self.path) - 1] == '/':
            #     glob_pattern = self.path + "*.py"
            #else:
            #     glob_pattern = self.path + "/*.py"
            #found_modules = glob.glob(glob_pattern)
            # Remove the remaining path names from the module, since 
            # We also have to add 1 to the index of the /
            #self.found_modules = [os.path.basename(mod) for mod in found_modules]
            
      def scan(self):
            if self.path[len(self.path) - 1] == '/':
                  py_pattern = self.path + "*.py"
                  pyc_pattern = self.path = "*.pyc"
                  pyo_pattern = self.path = "*.pyo"
            else:
                  py_pattern = self.path + "/*.py"
                  pyc_pattern = self.path + "/*.pyc"
                  pyo_pattern = self.path + "/*.pyo"
                  
            # Get globs
            found_py = glob.glob(py_pattern)
            found_pyc = glob.glob(pyc_pattern)
            found_pyo = glob.glob(pyo_pattern)
            
            # Convert to basename
            base_py = [os.path.basename(mod) for mod in found_py]
            base_pyc = [os.path.basename(mod) for mod in found_pyc]
            base_pyo = [os.path.basename(mod) for mod in found_pyo]
            
            stripped_py = [mod[0:mod.rfind(".py")] for mod in base_py]
            stripped_pyc = [mod[0:mod.rfind(".pyc")] for mod in base_pyc]
            stripped_pyo = [mod[0:mod.rfind(".pyo")] for mod in base_pyo]
            
            
            # Remove the remaining path names from the module, since 
            # We also have to add 1 to the index of the /
            self.found_modules = stripped_py
            for mod in stripped_pyc:
                  if mod not in self.found_modules:
                        if self.debug_output >= DEBUG_ALL:
                              print "Module %s does not have a corresponding source file" % mod
                        self.found_modules.append(mod)
                  elif self.debug_output >= DEBUG_ALL:
                        print "Module %s has a bytecode file" % mod
                        
            for mod in stripped_pyo:
                  if mod not in self.found_modules:
                        if self.debug_output >= DEBUG_ALL:
                              print "Module %s does not have a corresponding source file or standard bytecode file" % mod
                        self.found_modules.append(mod)
                  elif self.debug_output >= DEBUG_ALL:
                        print "Module %s has a bytecode file" % mode
            
      # Load modules                
      def load(self):
            if self.debug_output >= DEBUG_ALL:
                  print "Located all modules: %s" % self.found_modules
            for modname in self.found_modules:
                  if self.debug_output >= DEBUG_ALL:
                        print "Attempting to 'find' module: %s" % modname
                  file_handle, filename, description = imp.find_module(modname)
                  loaded_module = None
                  if not file_handle:
                        if self.debug_output >= DEBUG_ALL:
                              print "Failed 'finding' module (programming error or possible collision with builtin module): %s" % stripped_modname
                  else:
                        try:
                              loaded_module = imp.load_module(modname, file_handle, modname, description)
                        except:
                              # Close our file handle
                              file_handle.close()
                              # If we are not using fault tolerance, reraise
                              if self.debug_output >= DEBUG_ALL:
                                    print "Load failed on module: %s, attempting recovery" % modname
                                    print sys.exc_info()[0]
                              if not self.fault_tolerance:
                                    raise
                                          
                        file_handle.close()

                        
                  # If loaded module is None, something went wrong
                  if loaded_module:
                        self.dir_modules.append(loaded_module)
                              
class ModuleDirectoryScannerIterator:
      def __init__(self, parent):
            self.parent = parent
            self.ind = -1
      def next(self):
            self.ind = self.ind + 1
            if self.ind == len(self.parent.dir_modules):
                  raise StopIteration
            else:
                  return self.parent.dir_modules[self.ind]
            

Generated by  Doxygen 1.6.0   Back to index