diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py
index 422134a7c5..49b7f1221e 100644
--- a/lib/ansible/inventory/__init__.py
+++ b/lib/ansible/inventory/__init__.py
@@ -66,9 +66,6 @@ class Inventory(object):
                 host_list = host_list.split(",")
                 host_list = [ h for h in host_list if h and h.strip() ]
 
-        else:
-            utils.plugins.vars_loader.add_directory(self.basedir())
-
         if type(host_list) == list:
             all = Group('all')
             self.groups = [ all ]
@@ -80,6 +77,8 @@ class Inventory(object):
                     all.add_host(Host(x))
         elif os.path.exists(host_list):
             if os.path.isdir(host_list):
+                # Ensure basedir is inside the directory
+                self.host_list = os.path.join(self.host_list, "")
                 self.parser = InventoryDirectory(filename=host_list)
                 self.groups = self.parser.groups.values()
             elif utils.is_executable(host_list):
@@ -92,6 +91,8 @@ class Inventory(object):
                     self.groups = self.parser.groups.values()
                 else:
                     raise errors.AnsibleError("YAML inventory support is deprecated in 0.6 and removed in 0.7, see the migration script in examples/scripts in the git checkout")
+
+            utils.plugins.vars_loader.add_directory(self.basedir())
         else:
             raise errors.AnsibleError("Unable to find an inventory file, specify one with -i ?")
 
diff --git a/lib/ansible/inventory/dir.py b/lib/ansible/inventory/dir.py
index 389fd41805..b92ef4072d 100644
--- a/lib/ansible/inventory/dir.py
+++ b/lib/ansible/inventory/dir.py
@@ -39,6 +39,9 @@ class InventoryDirectory(object):
         for i in self.names:
             if i.endswith("~") or i.endswith(".orig") or i.endswith(".bak"):
                 continue
+            # These are things inside of an inventory basedir
+            if i in ("host_vars", "group_vars", "vars_plugins"):
+                continue
             fullpath = os.path.join(self.directory, i)
             if os.path.isdir(fullpath):
                 parser = InventoryDirectory(filename=fullpath)